path: root/mm/vmalloc.c (plain)
blob: 532310229a36ed1ed9c7ed5f241de0287c3330a0

1	/*
2	* linux/mm/vmalloc.c
3	*
4	* Copyright (C) 1993 Linus Torvalds
5	* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6	* SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7	* Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
8	* Numa awareness, Christoph Lameter, SGI, June 2005
9	*/
10
11	#include <linux/vmalloc.h>
12	#include <linux/mm.h>
13	#include <linux/module.h>
14	#include <linux/highmem.h>
15	#include <linux/sched.h>
16	#include <linux/slab.h>
17	#include <linux/spinlock.h>
18	#include <linux/interrupt.h>
19	#include <linux/proc_fs.h>
20	#include <linux/seq_file.h>
21	#include <linux/debugobjects.h>
22	#include <linux/kallsyms.h>
23	#include <linux/list.h>
24	#include <linux/notifier.h>
25	#include <linux/rbtree.h>
26	#include <linux/radix-tree.h>
27	#include <linux/rcupdate.h>
28	#include <linux/pfn.h>
29	#include <linux/kmemleak.h>
30	#include <linux/atomic.h>
31	#include <linux/compiler.h>
32	#include <linux/llist.h>
33	#include <linux/bitops.h>
34
35	#include <asm/uaccess.h>
36	#include <asm/tlbflush.h>
37	#include <asm/shmparam.h>
38
39	#include "internal.h"
40
41	struct vfree_deferred {
42	struct llist_head list;
43	struct work_struct wq;
44	};
45	static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
46
47	static void __vunmap(const void *, int);
48
49	static void free_work(struct work_struct *w)
50	{
51	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
52	struct llist_node *llnode = llist_del_all(&p->list);
53	while (llnode) {
54	void *p = llnode;
55	llnode = llist_next(llnode);
56	__vunmap(p, 1);
57	}
58	}
59
60	/*** Page table manipulation functions ***/
61
62	static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
63	{
64	pte_t *pte;
65
66	pte = pte_offset_kernel(pmd, addr);
67	do {
68	pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
69	WARN_ON(!pte_none(ptent) && !pte_present(ptent));
70	} while (pte++, addr += PAGE_SIZE, addr != end);
71	}
72
73	static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
74	{
75	pmd_t *pmd;
76	unsigned long next;
77
78	pmd = pmd_offset(pud, addr);
79	do {
80	next = pmd_addr_end(addr, end);
81	if (pmd_clear_huge(pmd))
82	continue;
83	if (pmd_none_or_clear_bad(pmd))
84	continue;
85	vunmap_pte_range(pmd, addr, next);
86	} while (pmd++, addr = next, addr != end);
87	}
88
89	static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
90	{
91	pud_t *pud;
92	unsigned long next;
93
94	pud = pud_offset(pgd, addr);
95	do {
96	next = pud_addr_end(addr, end);
97	if (pud_clear_huge(pud))
98	continue;
99	if (pud_none_or_clear_bad(pud))
100	continue;
101	vunmap_pmd_range(pud, addr, next);
102	} while (pud++, addr = next, addr != end);
103	}
104
105	static void vunmap_page_range(unsigned long addr, unsigned long end)
106	{
107	pgd_t *pgd;
108	unsigned long next;
109
110	BUG_ON(addr >= end);
111	pgd = pgd_offset_k(addr);
112	do {
113	next = pgd_addr_end(addr, end);
114	if (pgd_none_or_clear_bad(pgd))
115	continue;
116	vunmap_pud_range(pgd, addr, next);
117	} while (pgd++, addr = next, addr != end);
118	}
119
120	static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
121	unsigned long end, pgprot_t prot, struct page **pages, int *nr)
122	{
123	pte_t *pte;
124
125	/*
126	* nr is a running index into the array which helps higher level
127	* callers keep track of where we're up to.
128	*/
129
130	pte = pte_alloc_kernel(pmd, addr);
131	if (!pte)
132	return -ENOMEM;
133	do {
134	struct page *page = pages[*nr];
135
136	if (WARN_ON(!pte_none(*pte)))
137	return -EBUSY;
138	if (WARN_ON(!page))
139	return -ENOMEM;
140	set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
141	(*nr)++;
142	} while (pte++, addr += PAGE_SIZE, addr != end);
143	return 0;
144	}
145
146	static int vmap_pmd_range(pud_t *pud, unsigned long addr,
147	unsigned long end, pgprot_t prot, struct page **pages, int *nr)
148	{
149	pmd_t *pmd;
150	unsigned long next;
151
152	pmd = pmd_alloc(&init_mm, pud, addr);
153	if (!pmd)
154	return -ENOMEM;
155	do {
156	next = pmd_addr_end(addr, end);
157	if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
158	return -ENOMEM;
159	} while (pmd++, addr = next, addr != end);
160	return 0;
161	}
162
163	static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
164	unsigned long end, pgprot_t prot, struct page **pages, int *nr)
165	{
166	pud_t *pud;
167	unsigned long next;
168
169	pud = pud_alloc(&init_mm, pgd, addr);
170	if (!pud)
171	return -ENOMEM;
172	do {
173	next = pud_addr_end(addr, end);
174	if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
175	return -ENOMEM;
176	} while (pud++, addr = next, addr != end);
177	return 0;
178	}
179
180	/*
181	* Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
182	* will have pfns corresponding to the "pages" array.
183	*
184	* Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
185	*/
186	static int vmap_page_range_noflush(unsigned long start, unsigned long end,
187	pgprot_t prot, struct page **pages)
188	{
189	pgd_t *pgd;
190	unsigned long next;
191	unsigned long addr = start;
192	int err = 0;
193	int nr = 0;
194
195	BUG_ON(addr >= end);
196	pgd = pgd_offset_k(addr);
197	do {
198	next = pgd_addr_end(addr, end);
199	err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
200	if (err)
201	return err;
202	} while (pgd++, addr = next, addr != end);
203
204	return nr;
205	}
206
207	static int vmap_page_range(unsigned long start, unsigned long end,
208	pgprot_t prot, struct page **pages)
209	{
210	int ret;
211
212	ret = vmap_page_range_noflush(start, end, prot, pages);
213	flush_cache_vmap(start, end);
214	return ret;
215	}
216
217	int is_vmalloc_or_module_addr(const void *x)
218	{
219	/*
220	* ARM, x86-64 and sparc64 put modules in a special place,
221	* and fall back on vmalloc() if that fails. Others
222	* just put it in the vmalloc space.
223	*/
224	#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
225	unsigned long addr = (unsigned long)x;
226	if (addr >= MODULES_VADDR && addr < MODULES_END)
227	return 1;
228	#endif
229	return is_vmalloc_addr(x);
230	}
231
232	/*
233	* Walk a vmap address to the struct page it maps.
234	*/
235	struct page *vmalloc_to_page(const void *vmalloc_addr)
236	{
237	unsigned long addr = (unsigned long) vmalloc_addr;
238	struct page *page = NULL;
239	pgd_t *pgd = pgd_offset_k(addr);
240
241	/*
242	* XXX we might need to change this if we add VIRTUAL_BUG_ON for
243	* architectures that do not vmalloc module space
244	*/
245	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
246
247	/*
248	* Don't dereference bad PUD or PMD (below) entries. This will also
249	* identify huge mappings, which we may encounter on architectures
250	* that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be
251	* identified as vmalloc addresses by is_vmalloc_addr(), but are
252	* not [unambiguously] associated with a struct page, so there is
253	* no correct value to return for them.
254	*/
255	if (!pgd_none(*pgd)) {
256	pud_t *pud = pud_offset(pgd, addr);
257	#ifndef CONFIG_AMLOGIC_MODIFY
258	WARN_ON_ONCE(pud_bad(*pud));
259	#endif
260	if (!pud_none(*pud) && !pud_bad(*pud)) {
261	pmd_t *pmd = pmd_offset(pud, addr);
262	#ifndef CONFIG_AMLOGIC_MODIFY
263	WARN_ON_ONCE(pmd_bad(*pmd));
264	#endif
265	if (!pmd_none(*pmd) && !pmd_bad(*pmd)) {
266	pte_t *ptep, pte;
267
268	ptep = pte_offset_map(pmd, addr);
269	pte = *ptep;
270	if (pte_present(pte))
271	page = pte_page(pte);
272	pte_unmap(ptep);
273	}
274	}
275	}
276	return page;
277	}
278	EXPORT_SYMBOL(vmalloc_to_page);
279
280	/*
281	* Map a vmalloc()-space virtual address to the physical page frame number.
282	*/
283	unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
284	{
285	return page_to_pfn(vmalloc_to_page(vmalloc_addr));
286	}
287	EXPORT_SYMBOL(vmalloc_to_pfn);
288
289
290	/*** Global kva allocator ***/
291
292	#define VM_VM_AREA 0x04
293
294	static DEFINE_SPINLOCK(vmap_area_lock);
295	/* Export for kexec only */
296	LIST_HEAD(vmap_area_list);
297	static LLIST_HEAD(vmap_purge_list);
298	static struct rb_root vmap_area_root = RB_ROOT;
299
300	/* The vmap cache globals are protected by vmap_area_lock */
301	static struct rb_node *free_vmap_cache;
302	static unsigned long cached_hole_size;
303	static unsigned long cached_vstart;
304	static unsigned long cached_align;
305
306	static unsigned long vmap_area_pcpu_hole;
307
308	static struct vmap_area *__find_vmap_area(unsigned long addr)
309	{
310	struct rb_node *n = vmap_area_root.rb_node;
311
312	while (n) {
313	struct vmap_area *va;
314
315	va = rb_entry(n, struct vmap_area, rb_node);
316	if (addr < va->va_start)
317	n = n->rb_left;
318	else if (addr >= va->va_end)
319	n = n->rb_right;
320	else
321	return va;
322	}
323
324	return NULL;
325	}
326
327	static void __insert_vmap_area(struct vmap_area *va)
328	{
329	struct rb_node **p = &vmap_area_root.rb_node;
330	struct rb_node *parent = NULL;
331	struct rb_node *tmp;
332
333	while (*p) {
334	struct vmap_area *tmp_va;
335
336	parent = *p;
337	tmp_va = rb_entry(parent, struct vmap_area, rb_node);
338	if (va->va_start < tmp_va->va_end)
339	p = &(*p)->rb_left;
340	else if (va->va_end > tmp_va->va_start)
341	p = &(*p)->rb_right;
342	else
343	BUG();
344	}
345
346	rb_link_node(&va->rb_node, parent, p);
347	rb_insert_color(&va->rb_node, &vmap_area_root);
348
349	/* address-sort this list */
350	tmp = rb_prev(&va->rb_node);
351	if (tmp) {
352	struct vmap_area *prev;
353	prev = rb_entry(tmp, struct vmap_area, rb_node);
354	list_add_rcu(&va->list, &prev->list);
355	} else
356	list_add_rcu(&va->list, &vmap_area_list);
357	}
358
359	static void purge_vmap_area_lazy(void);
360
361	static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
362
363	#ifdef CONFIG_AMLOGIC_MODIFY
364	static void dump_vmalloc(void)
365	{
366	struct vmap_area *va, *next;
367
368	spin_lock(&vmap_area_lock);
369	list_for_each_entry_safe(va, next, &vmap_area_list, list) {
370	pr_info("%s, va:%lx-%lx, size:%08ld KB, alloc:%pf\n",
371	__func__, va->va_start, va->va_end,
372	(va->va_end - va->va_start) >> 10, va->vm->caller);
373	}
374	spin_unlock(&vmap_area_lock);
375	}
376	#endif
377
378	/*
379	* Allocate a region of KVA of the specified size and alignment, within the
380	* vstart and vend.
381	*/
382	static struct vmap_area *alloc_vmap_area(unsigned long size,
383	unsigned long align,
384	unsigned long vstart, unsigned long vend,
385	int node, gfp_t gfp_mask)
386	{
387	struct vmap_area *va;
388	struct rb_node *n;
389	unsigned long addr;
390	int purged = 0;
391	struct vmap_area *first;
392
393	BUG_ON(!size);
394	BUG_ON(offset_in_page(size));
395	BUG_ON(!is_power_of_2(align));
396
397	might_sleep_if(gfpflags_allow_blocking(gfp_mask));
398
399	va = kmalloc_node(sizeof(struct vmap_area),
400	gfp_mask & GFP_RECLAIM_MASK, node);
401	if (unlikely(!va))
402	return ERR_PTR(-ENOMEM);
403
404	/*
405	* Only scan the relevant parts containing pointers to other objects
406	* to avoid false negatives.
407	*/
408	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
409
410	retry:
411	spin_lock(&vmap_area_lock);
412	/*
413	* Invalidate cache if we have more permissive parameters.
414	* cached_hole_size notes the largest hole noticed _below_
415	* the vmap_area cached in free_vmap_cache: if size fits
416	* into that hole, we want to scan from vstart to reuse
417	* the hole instead of allocating above free_vmap_cache.
418	* Note that __free_vmap_area may update free_vmap_cache
419	* without updating cached_hole_size or cached_align.
420	*/
421	if (!free_vmap_cache ||
422	size < cached_hole_size ||
423	vstart < cached_vstart ||
424	align < cached_align) {
425	nocache:
426	cached_hole_size = 0;
427	free_vmap_cache = NULL;
428	}
429	/* record if we encounter less permissive parameters */
430	cached_vstart = vstart;
431	cached_align = align;
432
433	/* find starting point for our search */
434	if (free_vmap_cache) {
435	first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
436	addr = ALIGN(first->va_end, align);
437	if (addr < vstart)
438	goto nocache;
439	if (addr + size < addr)
440	goto overflow;
441
442	} else {
443	addr = ALIGN(vstart, align);
444	if (addr + size < addr)
445	goto overflow;
446
447	n = vmap_area_root.rb_node;
448	first = NULL;
449
450	while (n) {
451	struct vmap_area *tmp;
452	tmp = rb_entry(n, struct vmap_area, rb_node);
453	if (tmp->va_end >= addr) {
454	first = tmp;
455	if (tmp->va_start <= addr)
456	break;
457	n = n->rb_left;
458	} else
459	n = n->rb_right;
460	}
461
462	if (!first)
463	goto found;
464	}
465
466	/* from the starting point, walk areas until a suitable hole is found */
467	while (addr + size > first->va_start && addr + size <= vend) {
468	if (addr + cached_hole_size < first->va_start)
469	cached_hole_size = first->va_start - addr;
470	addr = ALIGN(first->va_end, align);
471	if (addr + size < addr)
472	goto overflow;
473
474	if (list_is_last(&first->list, &vmap_area_list))
475	goto found;
476
477	first = list_next_entry(first, list);
478	}
479
480	found:
481	/*
482	* Check also calculated address against the vstart,
483	* because it can be 0 because of big align request.
484	*/
485	if (addr + size > vend || addr < vstart)
486	goto overflow;
487
488	va->va_start = addr;
489	va->va_end = addr + size;
490	va->flags = 0;
491	__insert_vmap_area(va);
492	free_vmap_cache = &va->rb_node;
493	spin_unlock(&vmap_area_lock);
494
495	BUG_ON(!IS_ALIGNED(va->va_start, align));
496	BUG_ON(va->va_start < vstart);
497	BUG_ON(va->va_end > vend);
498
499	return va;
500
501	overflow:
502	spin_unlock(&vmap_area_lock);
503	if (!purged) {
504	purge_vmap_area_lazy();
505	purged = 1;
506	goto retry;
507	}
508
509	if (gfpflags_allow_blocking(gfp_mask)) {
510	unsigned long freed = 0;
511	blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
512	if (freed > 0) {
513	purged = 0;
514	goto retry;
515	}
516	}
517
518	#ifdef CONFIG_AMLOGIC_MODIFY
519	dump_vmalloc();
520	#endif
521	if (printk_ratelimit())
522	pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
523	size);
524	kfree(va);
525	return ERR_PTR(-EBUSY);
526	}
527
528	int register_vmap_purge_notifier(struct notifier_block *nb)
529	{
530	return blocking_notifier_chain_register(&vmap_notify_list, nb);
531	}
532	EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
533
534	int unregister_vmap_purge_notifier(struct notifier_block *nb)
535	{
536	return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
537	}
538	EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
539
540	static void __free_vmap_area(struct vmap_area *va)
541	{
542	BUG_ON(RB_EMPTY_NODE(&va->rb_node));
543
544	if (free_vmap_cache) {
545	if (va->va_end < cached_vstart) {
546	free_vmap_cache = NULL;
547	} else {
548	struct vmap_area *cache;
549	cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
550	if (va->va_start <= cache->va_start) {
551	free_vmap_cache = rb_prev(&va->rb_node);
552	/*
553	* We don't try to update cached_hole_size or
554	* cached_align, but it won't go very wrong.
555	*/
556	}
557	}
558	}
559	rb_erase(&va->rb_node, &vmap_area_root);
560	RB_CLEAR_NODE(&va->rb_node);
561	list_del_rcu(&va->list);
562
563	/*
564	* Track the highest possible candidate for pcpu area
565	* allocation. Areas outside of vmalloc area can be returned
566	* here too, consider only end addresses which fall inside
567	* vmalloc area proper.
568	*/
569	if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
570	vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
571
572	kfree_rcu(va, rcu_head);
573	}
574
575	/*
576	* Free a region of KVA allocated by alloc_vmap_area
577	*/
578	static void free_vmap_area(struct vmap_area *va)
579	{
580	spin_lock(&vmap_area_lock);
581	__free_vmap_area(va);
582	spin_unlock(&vmap_area_lock);
583	}
584
585	/*
586	* Clear the pagetable entries of a given vmap_area
587	*/
588	static void unmap_vmap_area(struct vmap_area *va)
589	{
590	vunmap_page_range(va->va_start, va->va_end);
591	}
592
593	static void vmap_debug_free_range(unsigned long start, unsigned long end)
594	{
595	/*
596	* Unmap page tables and force a TLB flush immediately if pagealloc
597	* debugging is enabled. This catches use after free bugs similarly to
598	* those in linear kernel virtual address space after a page has been
599	* freed.
600	*
601	* All the lazy freeing logic is still retained, in order to minimise
602	* intrusiveness of this debugging feature.
603	*
604	* This is going to be *slow* (linear kernel virtual address debugging
605	* doesn't do a broadcast TLB flush so it is a lot faster).
606	*/
607	if (debug_pagealloc_enabled()) {
608	vunmap_page_range(start, end);
609	flush_tlb_kernel_range(start, end);
610	}
611	}
612
613	/*
614	* lazy_max_pages is the maximum amount of virtual address space we gather up
615	* before attempting to purge with a TLB flush.
616	*
617	* There is a tradeoff here: a larger number will cover more kernel page tables
618	* and take slightly longer to purge, but it will linearly reduce the number of
619	* global TLB flushes that must be performed. It would seem natural to scale
620	* this number up linearly with the number of CPUs (because vmapping activity
621	* could also scale linearly with the number of CPUs), however it is likely
622	* that in practice, workloads might be constrained in other ways that mean
623	* vmap activity will not scale linearly with CPUs. Also, I want to be
624	* conservative and not introduce a big latency on huge systems, so go with
625	* a less aggressive log scale. It will still be an improvement over the old
626	* code, and it will be simple to change the scale factor if we find that it
627	* becomes a problem on bigger systems.
628	*/
629	static unsigned long lazy_max_pages(void)
630	{
631	unsigned int log;
632
633	log = fls(num_online_cpus());
634
635	return log * (32UL * 1024 * 1024 / PAGE_SIZE);
636	}
637
638	static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
639
640	/* for per-CPU blocks */
641	static void purge_fragmented_blocks_allcpus(void);
642
643	/*
644	* called before a call to iounmap() if the caller wants vm_area_struct's
645	* immediately freed.
646	*/
647	void set_iounmap_nonlazy(void)
648	{
649	atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
650	}
651
652	/*
653	* Purges all lazily-freed vmap areas.
654	*
655	* If sync is 0 then don't purge if there is already a purge in progress.
656	* If force_flush is 1, then flush kernel TLBs between *start and *end even
657	* if we found no lazy vmap areas to unmap (callers can use this to optimise
658	* their own TLB flushing).
659	* Returns with *start = min(*start, lowest purged address)
660	* *end = max(*end, highest purged address)
661	*/
662	static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
663	int sync, int force_flush)
664	{
665	static DEFINE_SPINLOCK(purge_lock);
666	struct llist_node *valist;
667	struct vmap_area *va;
668	struct vmap_area *n_va;
669	int nr = 0;
670
671	/*
672	* If sync is 0 but force_flush is 1, we'll go sync anyway but callers
673	* should not expect such behaviour. This just simplifies locking for
674	* the case that isn't actually used at the moment anyway.
675	*/
676	if (!sync && !force_flush) {
677	if (!spin_trylock(&purge_lock))
678	return;
679	} else
680	spin_lock(&purge_lock);
681
682	if (sync)
683	purge_fragmented_blocks_allcpus();
684
685	valist = llist_del_all(&vmap_purge_list);
686	llist_for_each_entry(va, valist, purge_list) {
687	if (va->va_start < *start)
688	*start = va->va_start;
689	if (va->va_end > *end)
690	*end = va->va_end;
691	nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
692	}
693
694	if (nr)
695	atomic_sub(nr, &vmap_lazy_nr);
696
697	if (nr || force_flush)
698	flush_tlb_kernel_range(*start, *end);
699
700	if (nr) {
701	spin_lock(&vmap_area_lock);
702	llist_for_each_entry_safe(va, n_va, valist, purge_list)
703	__free_vmap_area(va);
704	spin_unlock(&vmap_area_lock);
705	}
706	spin_unlock(&purge_lock);
707	}
708
709	/*
710	* Kick off a purge of the outstanding lazy areas. Don't bother if somebody
711	* is already purging.
712	*/
713	static void try_purge_vmap_area_lazy(void)
714	{
715	unsigned long start = ULONG_MAX, end = 0;
716
717	__purge_vmap_area_lazy(&start, &end, 0, 0);
718	}
719
720	/*
721	* Kick off a purge of the outstanding lazy areas.
722	*/
723	static void purge_vmap_area_lazy(void)
724	{
725	unsigned long start = ULONG_MAX, end = 0;
726
727	__purge_vmap_area_lazy(&start, &end, 1, 0);
728	}
729
730	/*
731	* Free a vmap area, caller ensuring that the area has been unmapped
732	* and flush_cache_vunmap had been called for the correct range
733	* previously.
734	*/
735	static void free_vmap_area_noflush(struct vmap_area *va)
736	{
737	int nr_lazy;
738
739	nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT,
740	&vmap_lazy_nr);
741
742	/* After this point, we may free va at any time */
743	llist_add(&va->purge_list, &vmap_purge_list);
744
745	if (unlikely(nr_lazy > lazy_max_pages()))
746	try_purge_vmap_area_lazy();
747	}
748
749	/*
750	* Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
751	* called for the correct range previously.
752	*/
753	static void free_unmap_vmap_area_noflush(struct vmap_area *va)
754	{
755	unmap_vmap_area(va);
756	free_vmap_area_noflush(va);
757	}
758
759	/*
760	* Free and unmap a vmap area
761	*/
762	static void free_unmap_vmap_area(struct vmap_area *va)
763	{
764	flush_cache_vunmap(va->va_start, va->va_end);
765	free_unmap_vmap_area_noflush(va);
766	}
767
768	static struct vmap_area *find_vmap_area(unsigned long addr)
769	{
770	struct vmap_area *va;
771
772	spin_lock(&vmap_area_lock);
773	va = __find_vmap_area(addr);
774	spin_unlock(&vmap_area_lock);
775
776	return va;
777	}
778
779	static void free_unmap_vmap_area_addr(unsigned long addr)
780	{
781	struct vmap_area *va;
782
783	va = find_vmap_area(addr);
784	BUG_ON(!va);
785	free_unmap_vmap_area(va);
786	}
787
788
789	/*** Per cpu kva allocator ***/
790
791	/*
792	* vmap space is limited especially on 32 bit architectures. Ensure there is
793	* room for at least 16 percpu vmap blocks per CPU.
794	*/
795	/*
796	* If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
797	* to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
798	* instead (we just need a rough idea)
799	*/
800	#if BITS_PER_LONG == 32
801	#define VMALLOC_SPACE (128UL*1024*1024)
802	#else
803	#define VMALLOC_SPACE (128UL*1024*1024*1024)
804	#endif
805
806	#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
807	#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
808	#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
809	#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
810	#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
811	#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
812	#define VMAP_BBMAP_BITS \
813	VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
814	VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
815	VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
816
817	#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
818
819	static bool vmap_initialized __read_mostly = false;
820
821	struct vmap_block_queue {
822	spinlock_t lock;
823	struct list_head free;
824	};
825
826	struct vmap_block {
827	spinlock_t lock;
828	struct vmap_area *va;
829	unsigned long free, dirty;
830	unsigned long dirty_min, dirty_max; /*< dirty range */
831	struct list_head free_list;
832	struct rcu_head rcu_head;
833	struct list_head purge;
834	};
835
836	/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
837	static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
838
839	/*
840	* Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
841	* in the free path. Could get rid of this if we change the API to return a
842	* "cookie" from alloc, to be passed to free. But no big deal yet.
843	*/
844	static DEFINE_SPINLOCK(vmap_block_tree_lock);
845	static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
846
847	/*
848	* We should probably have a fallback mechanism to allocate virtual memory
849	* out of partially filled vmap blocks. However vmap block sizing should be
850	* fairly reasonable according to the vmalloc size, so it shouldn't be a
851	* big problem.
852	*/
853
854	static unsigned long addr_to_vb_idx(unsigned long addr)
855	{
856	addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
857	addr /= VMAP_BLOCK_SIZE;
858	return addr;
859	}
860
861	static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
862	{
863	unsigned long addr;
864
865	addr = va_start + (pages_off << PAGE_SHIFT);
866	BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
867	return (void *)addr;
868	}
869
870	/**
871	* new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
872	* block. Of course pages number can't exceed VMAP_BBMAP_BITS
873	* @order: how many 2^order pages should be occupied in newly allocated block
874	* @gfp_mask: flags for the page level allocator
875	*
876	* Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
877	*/
878	static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
879	{
880	struct vmap_block_queue *vbq;
881	struct vmap_block *vb;
882	struct vmap_area *va;
883	unsigned long vb_idx;
884	int node, err;
885	void *vaddr;
886
887	node = numa_node_id();
888
889	vb = kmalloc_node(sizeof(struct vmap_block),
890	gfp_mask & GFP_RECLAIM_MASK, node);
891	if (unlikely(!vb))
892	return ERR_PTR(-ENOMEM);
893
894	va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
895	VMALLOC_START, VMALLOC_END,
896	node, gfp_mask);
897	if (IS_ERR(va)) {
898	kfree(vb);
899	return ERR_CAST(va);
900	}
901
902	err = radix_tree_preload(gfp_mask);
903	if (unlikely(err)) {
904	kfree(vb);
905	free_vmap_area(va);
906	return ERR_PTR(err);
907	}
908
909	vaddr = vmap_block_vaddr(va->va_start, 0);
910	spin_lock_init(&vb->lock);
911	vb->va = va;
912	/* At least something should be left free */
913	BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
914	vb->free = VMAP_BBMAP_BITS - (1UL << order);
915	vb->dirty = 0;
916	vb->dirty_min = VMAP_BBMAP_BITS;
917	vb->dirty_max = 0;
918	INIT_LIST_HEAD(&vb->free_list);
919
920	vb_idx = addr_to_vb_idx(va->va_start);
921	spin_lock(&vmap_block_tree_lock);
922	err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
923	spin_unlock(&vmap_block_tree_lock);
924	BUG_ON(err);
925	radix_tree_preload_end();
926
927	vbq = &get_cpu_var(vmap_block_queue);
928	spin_lock(&vbq->lock);
929	list_add_tail_rcu(&vb->free_list, &vbq->free);
930	spin_unlock(&vbq->lock);
931	put_cpu_var(vmap_block_queue);
932
933	return vaddr;
934	}
935
936	static void free_vmap_block(struct vmap_block *vb)
937	{
938	struct vmap_block *tmp;
939	unsigned long vb_idx;
940
941	vb_idx = addr_to_vb_idx(vb->va->va_start);
942	spin_lock(&vmap_block_tree_lock);
943	tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
944	spin_unlock(&vmap_block_tree_lock);
945	BUG_ON(tmp != vb);
946
947	free_vmap_area_noflush(vb->va);
948	kfree_rcu(vb, rcu_head);
949	}
950
951	static void purge_fragmented_blocks(int cpu)
952	{
953	LIST_HEAD(purge);
954	struct vmap_block *vb;
955	struct vmap_block *n_vb;
956	struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
957
958	rcu_read_lock();
959	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
960
961	if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
962	continue;
963
964	spin_lock(&vb->lock);
965	if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
966	vb->free = 0; /* prevent further allocs after releasing lock */
967	vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
968	vb->dirty_min = 0;
969	vb->dirty_max = VMAP_BBMAP_BITS;
970	spin_lock(&vbq->lock);
971	list_del_rcu(&vb->free_list);
972	spin_unlock(&vbq->lock);
973	spin_unlock(&vb->lock);
974	list_add_tail(&vb->purge, &purge);
975	} else
976	spin_unlock(&vb->lock);
977	}
978	rcu_read_unlock();
979
980	list_for_each_entry_safe(vb, n_vb, &purge, purge) {
981	list_del(&vb->purge);
982	free_vmap_block(vb);
983	}
984	}
985
986	static void purge_fragmented_blocks_allcpus(void)
987	{
988	int cpu;
989
990	for_each_possible_cpu(cpu)
991	purge_fragmented_blocks(cpu);
992	}
993
994	static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
995	{
996	struct vmap_block_queue *vbq;
997	struct vmap_block *vb;
998	void *vaddr = NULL;
999	unsigned int order;
1000
1001	BUG_ON(offset_in_page(size));
1002	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1003	if (WARN_ON(size == 0)) {
1004	/*
1005	* Allocating 0 bytes isn't what caller wants since
1006	* get_order(0) returns funny result. Just warn and terminate
1007	* early.
1008	*/
1009	return NULL;
1010	}
1011	order = get_order(size);
1012
1013	rcu_read_lock();
1014	vbq = &get_cpu_var(vmap_block_queue);
1015	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1016	unsigned long pages_off;
1017
1018	spin_lock(&vb->lock);
1019	if (vb->free < (1UL << order)) {
1020	spin_unlock(&vb->lock);
1021	continue;
1022	}
1023
1024	pages_off = VMAP_BBMAP_BITS - vb->free;
1025	vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
1026	vb->free -= 1UL << order;
1027	if (vb->free == 0) {
1028	spin_lock(&vbq->lock);
1029	list_del_rcu(&vb->free_list);
1030	spin_unlock(&vbq->lock);
1031	}
1032
1033	spin_unlock(&vb->lock);
1034	break;
1035	}
1036
1037	put_cpu_var(vmap_block_queue);
1038	rcu_read_unlock();
1039
1040	/* Allocate new block if nothing was found */
1041	if (!vaddr)
1042	vaddr = new_vmap_block(order, gfp_mask);
1043
1044	return vaddr;
1045	}
1046
1047	static void vb_free(const void *addr, unsigned long size)
1048	{
1049	unsigned long offset;
1050	unsigned long vb_idx;
1051	unsigned int order;
1052	struct vmap_block *vb;
1053
1054	BUG_ON(offset_in_page(size));
1055	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
1056
1057	flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
1058
1059	order = get_order(size);
1060
1061	offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
1062	offset >>= PAGE_SHIFT;
1063
1064	vb_idx = addr_to_vb_idx((unsigned long)addr);
1065	rcu_read_lock();
1066	vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1067	rcu_read_unlock();
1068	BUG_ON(!vb);
1069
1070	vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
1071
1072	spin_lock(&vb->lock);
1073
1074	/* Expand dirty range */
1075	vb->dirty_min = min(vb->dirty_min, offset);
1076	vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
1077
1078	vb->dirty += 1UL << order;
1079	if (vb->dirty == VMAP_BBMAP_BITS) {
1080	BUG_ON(vb->free);
1081	spin_unlock(&vb->lock);
1082	free_vmap_block(vb);
1083	} else
1084	spin_unlock(&vb->lock);
1085	}
1086
1087	/**
1088	* vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1089	*
1090	* The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1091	* to amortize TLB flushing overheads. What this means is that any page you
1092	* have now, may, in a former life, have been mapped into kernel virtual
1093	* address by the vmap layer and so there might be some CPUs with TLB entries
1094	* still referencing that page (additional to the regular 1:1 kernel mapping).
1095	*
1096	* vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1097	* be sure that none of the pages we have control over will have any aliases
1098	* from the vmap layer.
1099	*/
1100	void vm_unmap_aliases(void)
1101	{
1102	unsigned long start = ULONG_MAX, end = 0;
1103	int cpu;
1104	int flush = 0;
1105
1106	if (unlikely(!vmap_initialized))
1107	return;
1108
1109	for_each_possible_cpu(cpu) {
1110	struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1111	struct vmap_block *vb;
1112
1113	rcu_read_lock();
1114	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1115	spin_lock(&vb->lock);
1116	if (vb->dirty) {
1117	unsigned long va_start = vb->va->va_start;
1118	unsigned long s, e;
1119
1120	s = va_start + (vb->dirty_min << PAGE_SHIFT);
1121	e = va_start + (vb->dirty_max << PAGE_SHIFT);
1122
1123	start = min(s, start);
1124	end = max(e, end);
1125
1126	flush = 1;
1127	}
1128	spin_unlock(&vb->lock);
1129	}
1130	rcu_read_unlock();
1131	}
1132
1133	__purge_vmap_area_lazy(&start, &end, 1, flush);
1134	}
1135	EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1136
1137	/**
1138	* vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1139	* @mem: the pointer returned by vm_map_ram
1140	* @count: the count passed to that vm_map_ram call (cannot unmap partial)
1141	*/
1142	void vm_unmap_ram(const void *mem, unsigned int count)
1143	{
1144	unsigned long size = (unsigned long)count << PAGE_SHIFT;
1145	unsigned long addr = (unsigned long)mem;
1146
1147	BUG_ON(!addr);
1148	BUG_ON(addr < VMALLOC_START);
1149	BUG_ON(addr > VMALLOC_END);
1150	BUG_ON(!PAGE_ALIGNED(addr));
1151
1152	debug_check_no_locks_freed(mem, size);
1153	vmap_debug_free_range(addr, addr+size);
1154
1155	if (likely(count <= VMAP_MAX_ALLOC))
1156	vb_free(mem, size);
1157	else
1158	free_unmap_vmap_area_addr(addr);
1159	}
1160	EXPORT_SYMBOL(vm_unmap_ram);
1161
1162	/**
1163	* vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1164	* @pages: an array of pointers to the pages to be mapped
1165	* @count: number of pages
1166	* @node: prefer to allocate data structures on this node
1167	* @prot: memory protection to use. PAGE_KERNEL for regular RAM
1168	*
1169	* If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1170	* faster than vmap so it's good. But if you mix long-life and short-life
1171	* objects with vm_map_ram(), it could consume lots of address space through
1172	* fragmentation (especially on a 32bit machine). You could see failures in
1173	* the end. Please use this function for short-lived objects.
1174	*
1175	* Returns: a pointer to the address that has been mapped, or %NULL on failure
1176	*/
1177	void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
1178	{
1179	unsigned long size = (unsigned long)count << PAGE_SHIFT;
1180	unsigned long addr;
1181	void *mem;
1182
1183	if (likely(count <= VMAP_MAX_ALLOC)) {
1184	mem = vb_alloc(size, GFP_KERNEL);
1185	if (IS_ERR(mem))
1186	return NULL;
1187	addr = (unsigned long)mem;
1188	} else {
1189	struct vmap_area *va;
1190	va = alloc_vmap_area(size, PAGE_SIZE,
1191	VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1192	if (IS_ERR(va))
1193	return NULL;
1194
1195	addr = va->va_start;
1196	mem = (void *)addr;
1197	}
1198	if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
1199	vm_unmap_ram(mem, count);
1200	return NULL;
1201	}
1202	return mem;
1203	}
1204	EXPORT_SYMBOL(vm_map_ram);
1205
1206	static struct vm_struct *vmlist __initdata;
1207	/**
1208	* vm_area_add_early - add vmap area early during boot
1209	* @vm: vm_struct to add
1210	*
1211	* This function is used to add fixed kernel vm area to vmlist before
1212	* vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
1213	* should contain proper values and the other fields should be zero.
1214	*
1215	* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1216	*/
1217	void __init vm_area_add_early(struct vm_struct *vm)
1218	{
1219	struct vm_struct *tmp, **p;
1220
1221	BUG_ON(vmap_initialized);
1222	for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1223	if (tmp->addr >= vm->addr) {
1224	BUG_ON(tmp->addr < vm->addr + vm->size);
1225	break;
1226	} else
1227	BUG_ON(tmp->addr + tmp->size > vm->addr);
1228	}
1229	vm->next = *p;
1230	*p = vm;
1231	}
1232
1233	/**
1234	* vm_area_register_early - register vmap area early during boot
1235	* @vm: vm_struct to register
1236	* @align: requested alignment
1237	*
1238	* This function is used to register kernel vm area before
1239	* vmalloc_init() is called. @vm->size and @vm->flags should contain
1240	* proper values on entry and other fields should be zero. On return,
1241	* vm->addr contains the allocated address.
1242	*
1243	* DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1244	*/
1245	void __init vm_area_register_early(struct vm_struct *vm, size_t align)
1246	{
1247	static size_t vm_init_off __initdata;
1248	unsigned long addr;
1249
1250	addr = ALIGN(VMALLOC_START + vm_init_off, align);
1251	vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1252
1253	vm->addr = (void *)addr;
1254
1255	vm_area_add_early(vm);
1256	}
1257
1258	void __init vmalloc_init(void)
1259	{
1260	struct vmap_area *va;
1261	struct vm_struct *tmp;
1262	int i;
1263
1264	for_each_possible_cpu(i) {
1265	struct vmap_block_queue *vbq;
1266	struct vfree_deferred *p;
1267
1268	vbq = &per_cpu(vmap_block_queue, i);
1269	spin_lock_init(&vbq->lock);
1270	INIT_LIST_HEAD(&vbq->free);
1271	p = &per_cpu(vfree_deferred, i);
1272	init_llist_head(&p->list);
1273	INIT_WORK(&p->wq, free_work);
1274	}
1275
1276	/* Import existing vmlist entries. */
1277	for (tmp = vmlist; tmp; tmp = tmp->next) {
1278	va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
1279	va->flags = VM_VM_AREA;
1280	va->va_start = (unsigned long)tmp->addr;
1281	va->va_end = va->va_start + tmp->size;
1282	va->vm = tmp;
1283	__insert_vmap_area(va);
1284	}
1285
1286	vmap_area_pcpu_hole = VMALLOC_END;
1287
1288	vmap_initialized = true;
1289	}
1290
1291	/**
1292	* map_kernel_range_noflush - map kernel VM area with the specified pages
1293	* @addr: start of the VM area to map
1294	* @size: size of the VM area to map
1295	* @prot: page protection flags to use
1296	* @pages: pages to map
1297	*
1298	* Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
1299	* specify should have been allocated using get_vm_area() and its
1300	* friends.
1301	*
1302	* NOTE:
1303	* This function does NOT do any cache flushing. The caller is
1304	* responsible for calling flush_cache_vmap() on to-be-mapped areas
1305	* before calling this function.
1306	*
1307	* RETURNS:
1308	* The number of pages mapped on success, -errno on failure.
1309	*/
1310	int map_kernel_range_noflush(unsigned long addr, unsigned long size,
1311	pgprot_t prot, struct page **pages)
1312	{
1313	return vmap_page_range_noflush(addr, addr + size, prot, pages);
1314	}
1315
1316	/**
1317	* unmap_kernel_range_noflush - unmap kernel VM area
1318	* @addr: start of the VM area to unmap
1319	* @size: size of the VM area to unmap
1320	*
1321	* Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
1322	* specify should have been allocated using get_vm_area() and its
1323	* friends.
1324	*
1325	* NOTE:
1326	* This function does NOT do any cache flushing. The caller is
1327	* responsible for calling flush_cache_vunmap() on to-be-mapped areas
1328	* before calling this function and flush_tlb_kernel_range() after.
1329	*/
1330	void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
1331	{
1332	vunmap_page_range(addr, addr + size);
1333	}
1334	EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
1335
1336	/**
1337	* unmap_kernel_range - unmap kernel VM area and flush cache and TLB
1338	* @addr: start of the VM area to unmap
1339	* @size: size of the VM area to unmap
1340	*
1341	* Similar to unmap_kernel_range_noflush() but flushes vcache before
1342	* the unmapping and tlb after.
1343	*/
1344	void unmap_kernel_range(unsigned long addr, unsigned long size)
1345	{
1346	unsigned long end = addr + size;
1347
1348	flush_cache_vunmap(addr, end);
1349	vunmap_page_range(addr, end);
1350	flush_tlb_kernel_range(addr, end);
1351	}
1352	EXPORT_SYMBOL_GPL(unmap_kernel_range);
1353
1354	int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
1355	{
1356	unsigned long addr = (unsigned long)area->addr;
1357	unsigned long end = addr + get_vm_area_size(area);
1358	int err;
1359
1360	err = vmap_page_range(addr, end, prot, pages);
1361
1362	return err > 0 ? 0 : err;
1363	}
1364	EXPORT_SYMBOL_GPL(map_vm_area);
1365
1366	static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
1367	unsigned long flags, const void *caller)
1368	{
1369	spin_lock(&vmap_area_lock);
1370	vm->flags = flags;
1371	vm->addr = (void *)va->va_start;
1372	vm->size = va->va_end - va->va_start;
1373	vm->caller = caller;
1374	va->vm = vm;
1375	va->flags |= VM_VM_AREA;
1376	spin_unlock(&vmap_area_lock);
1377	}
1378
1379	static void clear_vm_uninitialized_flag(struct vm_struct *vm)
1380	{
1381	/*
1382	* Before removing VM_UNINITIALIZED,
1383	* we should make sure that vm has proper values.
1384	* Pair with smp_rmb() in show_numa_info().
1385	*/
1386	smp_wmb();
1387	vm->flags &= ~VM_UNINITIALIZED;
1388	}
1389
1390	#ifdef CONFIG_AMLOGIC_VMAP
1391	struct vm_struct *__get_vm_area_node(unsigned long size,
1392	unsigned long align, unsigned long flags, unsigned long start,
1393	unsigned long end, int node, gfp_t gfp_mask, const void *caller)
1394	#else
1395	static struct vm_struct *__get_vm_area_node(unsigned long size,
1396	unsigned long align, unsigned long flags, unsigned long start,
1397	unsigned long end, int node, gfp_t gfp_mask, const void *caller)
1398	#endif
1399	{
1400	struct vmap_area *va;
1401	struct vm_struct *area;
1402
1403	BUG_ON(in_interrupt());
1404	size = PAGE_ALIGN(size);
1405	if (unlikely(!size))
1406	return NULL;
1407
1408	if (flags & VM_IOREMAP)
1409	align = 1ul << clamp_t(int, get_count_order_long(size),
1410	PAGE_SHIFT, IOREMAP_MAX_ORDER);
1411
1412	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
1413	if (unlikely(!area))
1414	return NULL;
1415
1416	if (!(flags & VM_NO_GUARD))
1417	size += PAGE_SIZE;
1418
1419	va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
1420	if (IS_ERR(va)) {
1421	kfree(area);
1422	return NULL;
1423	}
1424
1425	setup_vmalloc_vm(area, va, flags, caller);
1426
1427	return area;
1428	}
1429
1430	struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
1431	unsigned long start, unsigned long end)
1432	{
1433	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1434	GFP_KERNEL, __builtin_return_address(0));
1435	}
1436	EXPORT_SYMBOL_GPL(__get_vm_area);
1437
1438	struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
1439	unsigned long start, unsigned long end,
1440	const void *caller)
1441	{
1442	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1443	GFP_KERNEL, caller);
1444	}
1445
1446	/**
1447	* get_vm_area - reserve a contiguous kernel virtual area
1448	* @size: size of the area
1449	* @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
1450	*
1451	* Search an area of @size in the kernel virtual mapping area,
1452	* and reserved it for out purposes. Returns the area descriptor
1453	* on success or %NULL on failure.
1454	*/
1455	struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
1456	{
1457	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
1458	NUMA_NO_NODE, GFP_KERNEL,
1459	__builtin_return_address(0));
1460	}
1461
1462	struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
1463	const void *caller)
1464	{
1465	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
1466	NUMA_NO_NODE, GFP_KERNEL, caller);
1467	}
1468
1469	/**
1470	* find_vm_area - find a continuous kernel virtual area
1471	* @addr: base address
1472	*
1473	* Search for the kernel VM area starting at @addr, and return it.
1474	* It is up to the caller to do all required locking to keep the returned
1475	* pointer valid.
1476	*/
1477	struct vm_struct *find_vm_area(const void *addr)
1478	{
1479	struct vmap_area *va;
1480
1481	va = find_vmap_area((unsigned long)addr);
1482	if (va && va->flags & VM_VM_AREA)
1483	return va->vm;
1484
1485	return NULL;
1486	}
1487
1488	/**
1489	* remove_vm_area - find and remove a continuous kernel virtual area
1490	* @addr: base address
1491	*
1492	* Search for the kernel VM area starting at @addr, and remove it.
1493	* This function returns the found VM area, but using it is NOT safe
1494	* on SMP machines, except for its size or flags.
1495	*/
1496	struct vm_struct *remove_vm_area(const void *addr)
1497	{
1498	struct vmap_area *va;
1499
1500	va = find_vmap_area((unsigned long)addr);
1501	if (va && va->flags & VM_VM_AREA) {
1502	struct vm_struct *vm = va->vm;
1503
1504	spin_lock(&vmap_area_lock);
1505	va->vm = NULL;
1506	va->flags &= ~VM_VM_AREA;
1507	spin_unlock(&vmap_area_lock);
1508
1509	vmap_debug_free_range(va->va_start, va->va_end);
1510	kasan_free_shadow(vm);
1511	free_unmap_vmap_area(va);
1512
1513	return vm;
1514	}
1515	return NULL;
1516	}
1517
1518	static void __vunmap(const void *addr, int deallocate_pages)
1519	{
1520	struct vm_struct *area;
1521
1522	if (!addr)
1523	return;
1524
1525	if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
1526	addr))
1527	return;
1528
1529	area = find_vmap_area((unsigned long)addr)->vm;
1530	if (unlikely(!area)) {
1531	WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
1532	addr);
1533	return;
1534	}
1535
1536	debug_check_no_locks_freed(addr, get_vm_area_size(area));
1537	debug_check_no_obj_freed(addr, get_vm_area_size(area));
1538
1539	remove_vm_area(addr);
1540	if (deallocate_pages) {
1541	int i;
1542
1543	for (i = 0; i < area->nr_pages; i++) {
1544	struct page *page = area->pages[i];
1545
1546	BUG_ON(!page);
1547	__free_pages(page, 0);
1548	}
1549
1550	kvfree(area->pages);
1551	}
1552
1553	kfree(area);
1554	return;
1555	}
1556
1557	/**
1558	* vfree - release memory allocated by vmalloc()
1559	* @addr: memory base address
1560	*
1561	* Free the virtually continuous memory area starting at @addr, as
1562	* obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
1563	* NULL, no operation is performed.
1564	*
1565	* Must not be called in NMI context (strictly speaking, only if we don't
1566	* have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
1567	* conventions for vfree() arch-depenedent would be a really bad idea)
1568	*
1569	* NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
1570	*/
1571	void vfree(const void *addr)
1572	{
1573	BUG_ON(in_nmi());
1574
1575	kmemleak_free(addr);
1576
1577	if (!addr)
1578	return;
1579	if (unlikely(in_interrupt())) {
1580	struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred);
1581	if (llist_add((struct llist_node *)addr, &p->list))
1582	schedule_work(&p->wq);
1583	} else
1584	__vunmap(addr, 1);
1585	}
1586	EXPORT_SYMBOL(vfree);
1587
1588	/**
1589	* vunmap - release virtual mapping obtained by vmap()
1590	* @addr: memory base address
1591	*
1592	* Free the virtually contiguous memory area starting at @addr,
1593	* which was created from the page array passed to vmap().
1594	*
1595	* Must not be called in interrupt context.
1596	*/
1597	void vunmap(const void *addr)
1598	{
1599	BUG_ON(in_interrupt());
1600	might_sleep();
1601	if (addr)
1602	__vunmap(addr, 0);
1603	}
1604	EXPORT_SYMBOL(vunmap);
1605
1606	/**
1607	* vmap - map an array of pages into virtually contiguous space
1608	* @pages: array of page pointers
1609	* @count: number of pages to map
1610	* @flags: vm_area->flags
1611	* @prot: page protection for the mapping
1612	*
1613	* Maps @count pages from @pages into contiguous kernel virtual
1614	* space.
1615	*/
1616	void *vmap(struct page **pages, unsigned int count,
1617	unsigned long flags, pgprot_t prot)
1618	{
1619	struct vm_struct *area;
1620	unsigned long size; /* In bytes */
1621
1622	might_sleep();
1623
1624	if (count > totalram_pages)
1625	return NULL;
1626
1627	size = (unsigned long)count << PAGE_SHIFT;
1628	area = get_vm_area_caller(size, flags, __builtin_return_address(0));
1629	if (!area)
1630	return NULL;
1631
1632	if (map_vm_area(area, prot, pages)) {
1633	vunmap(area->addr);
1634	return NULL;
1635	}
1636
1637	return area->addr;
1638	}
1639	EXPORT_SYMBOL(vmap);
1640
1641	static void *__vmalloc_node(unsigned long size, unsigned long align,
1642	gfp_t gfp_mask, pgprot_t prot,
1643	int node, const void *caller);
1644	static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
1645	pgprot_t prot, int node)
1646	{
1647	struct page **pages;
1648	unsigned int nr_pages, array_size, i;
1649	const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
1650	const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
1651
1652	nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
1653	array_size = (nr_pages * sizeof(struct page *));
1654
1655	area->nr_pages = nr_pages;
1656	/* Please note that the recursion is strictly bounded. */
1657	if (array_size > PAGE_SIZE) {
1658	pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
1659	PAGE_KERNEL, node, area->caller);
1660	} else {
1661	pages = kmalloc_node(array_size, nested_gfp, node);
1662	}
1663	area->pages = pages;
1664	if (!area->pages) {
1665	remove_vm_area(area->addr);
1666	kfree(area);
1667	return NULL;
1668	}
1669
1670	for (i = 0; i < area->nr_pages; i++) {
1671	struct page *page;
1672
1673	if (node == NUMA_NO_NODE)
1674	page = alloc_page(alloc_mask);
1675	else
1676	page = alloc_pages_node(node, alloc_mask, 0);
1677
1678	if (unlikely(!page)) {
1679	/* Successfully allocated i pages, free them in __vunmap() */
1680	area->nr_pages = i;
1681	goto fail;
1682	}
1683	area->pages[i] = page;
1684	if (gfpflags_allow_blocking(gfp_mask))
1685	cond_resched();
1686	}
1687
1688	if (map_vm_area(area, prot, pages))
1689	goto fail;
1690	return area->addr;
1691
1692	fail:
1693	warn_alloc(gfp_mask,
1694	"vmalloc: allocation failure, allocated %ld of %ld bytes",
1695	(area->nr_pages*PAGE_SIZE), area->size);
1696	vfree(area->addr);
1697	return NULL;
1698	}
1699
1700	/**
1701	* __vmalloc_node_range - allocate virtually contiguous memory
1702	* @size: allocation size
1703	* @align: desired alignment
1704	* @start: vm area range start
1705	* @end: vm area range end
1706	* @gfp_mask: flags for the page level allocator
1707	* @prot: protection mask for the allocated pages
1708	* @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
1709	* @node: node to use for allocation or NUMA_NO_NODE
1710	* @caller: caller's return address
1711	*
1712	* Allocate enough pages to cover @size from the page level
1713	* allocator with @gfp_mask flags. Map them into contiguous
1714	* kernel virtual space, using a pagetable protection of @prot.
1715	*/
1716	void *__vmalloc_node_range(unsigned long size, unsigned long align,
1717	unsigned long start, unsigned long end, gfp_t gfp_mask,
1718	pgprot_t prot, unsigned long vm_flags, int node,
1719	const void *caller)
1720	{
1721	struct vm_struct *area;
1722	void *addr;
1723	unsigned long real_size = size;
1724
1725	size = PAGE_ALIGN(size);
1726	if (!size || (size >> PAGE_SHIFT) > totalram_pages)
1727	goto fail;
1728
1729	area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
1730	vm_flags, start, end, node, gfp_mask, caller);
1731	if (!area)
1732	goto fail;
1733
1734	addr = __vmalloc_area_node(area, gfp_mask, prot, node);
1735	if (!addr)
1736	return NULL;
1737
1738	/*
1739	* In this function, newly allocated vm_struct has VM_UNINITIALIZED
1740	* flag. It means that vm_struct is not fully initialized.
1741	* Now, it is fully initialized, so remove this flag here.
1742	*/
1743	clear_vm_uninitialized_flag(area);
1744
1745	/*
1746	* A ref_count = 2 is needed because vm_struct allocated in
1747	* __get_vm_area_node() contains a reference to the virtual address of
1748	* the vmalloc'ed block.
1749	*/
1750	kmemleak_alloc(addr, real_size, 2, gfp_mask);
1751
1752	return addr;
1753
1754	fail:
1755	warn_alloc(gfp_mask,
1756	"vmalloc: allocation failure: %lu bytes", real_size);
1757	return NULL;
1758	}
1759
1760	/**
1761	* __vmalloc_node - allocate virtually contiguous memory
1762	* @size: allocation size
1763	* @align: desired alignment
1764	* @gfp_mask: flags for the page level allocator
1765	* @prot: protection mask for the allocated pages
1766	* @node: node to use for allocation or NUMA_NO_NODE
1767	* @caller: caller's return address
1768	*
1769	* Allocate enough pages to cover @size from the page level
1770	* allocator with @gfp_mask flags. Map them into contiguous
1771	* kernel virtual space, using a pagetable protection of @prot.
1772	*/
1773	static void *__vmalloc_node(unsigned long size, unsigned long align,
1774	gfp_t gfp_mask, pgprot_t prot,
1775	int node, const void *caller)
1776	{
1777	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
1778	gfp_mask, prot, 0, node, caller);
1779	}
1780
1781	void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
1782	{
1783	return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
1784	__builtin_return_address(0));
1785	}
1786	EXPORT_SYMBOL(__vmalloc);
1787
1788	static inline void *__vmalloc_node_flags(unsigned long size,
1789	int node, gfp_t flags)
1790	{
1791	return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
1792	node, __builtin_return_address(0));
1793	}
1794
1795	/**
1796	* vmalloc - allocate virtually contiguous memory
1797	* @size: allocation size
1798	* Allocate enough pages to cover @size from the page level
1799	* allocator and map them into contiguous kernel virtual space.
1800	*
1801	* For tight control over page level allocator and protection flags
1802	* use __vmalloc() instead.
1803	*/
1804	void *vmalloc(unsigned long size)
1805	{
1806	return __vmalloc_node_flags(size, NUMA_NO_NODE,
1807	GFP_KERNEL | __GFP_HIGHMEM);
1808	}
1809	EXPORT_SYMBOL(vmalloc);
1810
1811	/**
1812	* vzalloc - allocate virtually contiguous memory with zero fill
1813	* @size: allocation size
1814	* Allocate enough pages to cover @size from the page level
1815	* allocator and map them into contiguous kernel virtual space.
1816	* The memory allocated is set to zero.
1817	*
1818	* For tight control over page level allocator and protection flags
1819	* use __vmalloc() instead.
1820	*/
1821	void *vzalloc(unsigned long size)
1822	{
1823	return __vmalloc_node_flags(size, NUMA_NO_NODE,
1824	GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1825	}
1826	EXPORT_SYMBOL(vzalloc);
1827
1828	/**
1829	* vmalloc_user - allocate zeroed virtually contiguous memory for userspace
1830	* @size: allocation size
1831	*
1832	* The resulting memory area is zeroed so it can be mapped to userspace
1833	* without leaking data.
1834	*/
1835	void *vmalloc_user(unsigned long size)
1836	{
1837	struct vm_struct *area;
1838	void *ret;
1839
1840	ret = __vmalloc_node(size, SHMLBA,
1841	GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
1842	PAGE_KERNEL, NUMA_NO_NODE,
1843	__builtin_return_address(0));
1844	if (ret) {
1845	area = find_vm_area(ret);
1846	area->flags |= VM_USERMAP;
1847	}
1848	return ret;
1849	}
1850	EXPORT_SYMBOL(vmalloc_user);
1851
1852	/**
1853	* vmalloc_node - allocate memory on a specific node
1854	* @size: allocation size
1855	* @node: numa node
1856	*
1857	* Allocate enough pages to cover @size from the page level
1858	* allocator and map them into contiguous kernel virtual space.
1859	*
1860	* For tight control over page level allocator and protection flags
1861	* use __vmalloc() instead.
1862	*/
1863	void *vmalloc_node(unsigned long size, int node)
1864	{
1865	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
1866	node, __builtin_return_address(0));
1867	}
1868	EXPORT_SYMBOL(vmalloc_node);
1869
1870	/**
1871	* vzalloc_node - allocate memory on a specific node with zero fill
1872	* @size: allocation size
1873	* @node: numa node
1874	*
1875	* Allocate enough pages to cover @size from the page level
1876	* allocator and map them into contiguous kernel virtual space.
1877	* The memory allocated is set to zero.
1878	*
1879	* For tight control over page level allocator and protection flags
1880	* use __vmalloc_node() instead.
1881	*/
1882	void *vzalloc_node(unsigned long size, int node)
1883	{
1884	return __vmalloc_node_flags(size, node,
1885	GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1886	}
1887	EXPORT_SYMBOL(vzalloc_node);
1888
1889	#ifndef PAGE_KERNEL_EXEC
1890	# define PAGE_KERNEL_EXEC PAGE_KERNEL
1891	#endif
1892
1893	/**
1894	* vmalloc_exec - allocate virtually contiguous, executable memory
1895	* @size: allocation size
1896	*
1897	* Kernel-internal function to allocate enough pages to cover @size
1898	* the page level allocator and map them into contiguous and
1899	* executable kernel virtual space.
1900	*
1901	* For tight control over page level allocator and protection flags
1902	* use __vmalloc() instead.
1903	*/
1904
1905	void *vmalloc_exec(unsigned long size)
1906	{
1907	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
1908	NUMA_NO_NODE, __builtin_return_address(0));
1909	}
1910
1911	#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
1912	#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
1913	#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
1914	#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
1915	#else
1916	#define GFP_VMALLOC32 GFP_KERNEL
1917	#endif
1918
1919	/**
1920	* vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
1921	* @size: allocation size
1922	*
1923	* Allocate enough 32bit PA addressable pages to cover @size from the
1924	* page level allocator and map them into contiguous kernel virtual space.
1925	*/
1926	void *vmalloc_32(unsigned long size)
1927	{
1928	return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
1929	NUMA_NO_NODE, __builtin_return_address(0));
1930	}
1931	EXPORT_SYMBOL(vmalloc_32);
1932
1933	/**
1934	* vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
1935	* @size: allocation size
1936	*
1937	* The resulting memory area is 32bit addressable and zeroed so it can be
1938	* mapped to userspace without leaking data.
1939	*/
1940	void *vmalloc_32_user(unsigned long size)
1941	{
1942	struct vm_struct *area;
1943	void *ret;
1944
1945	ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
1946	NUMA_NO_NODE, __builtin_return_address(0));
1947	if (ret) {
1948	area = find_vm_area(ret);
1949	area->flags |= VM_USERMAP;
1950	}
1951	return ret;
1952	}
1953	EXPORT_SYMBOL(vmalloc_32_user);
1954
1955	/*
1956	* small helper routine , copy contents to buf from addr.
1957	* If the page is not present, fill zero.
1958	*/
1959
1960	static int aligned_vread(char *buf, char *addr, unsigned long count)
1961	{
1962	struct page *p;
1963	int copied = 0;
1964
1965	while (count) {
1966	unsigned long offset, length;
1967
1968	offset = offset_in_page(addr);
1969	length = PAGE_SIZE - offset;
1970	if (length > count)
1971	length = count;
1972	p = vmalloc_to_page(addr);
1973	/*
1974	* To do safe access to this _mapped_ area, we need
1975	* lock. But adding lock here means that we need to add
1976	* overhead of vmalloc()/vfree() calles for this _debug_
1977	* interface, rarely used. Instead of that, we'll use
1978	* kmap() and get small overhead in this access function.
1979	*/
1980	if (p) {
1981	/*
1982	* we can expect USER0 is not used (see vread/vwrite's
1983	* function description)
1984	*/
1985	void *map = kmap_atomic(p);
1986	memcpy(buf, map + offset, length);
1987	kunmap_atomic(map);
1988	} else
1989	memset(buf, 0, length);
1990
1991	addr += length;
1992	buf += length;
1993	copied += length;
1994	count -= length;
1995	}
1996	return copied;
1997	}
1998
1999	static int aligned_vwrite(char *buf, char *addr, unsigned long count)
2000	{
2001	struct page *p;
2002	int copied = 0;
2003
2004	while (count) {
2005	unsigned long offset, length;
2006
2007	offset = offset_in_page(addr);
2008	length = PAGE_SIZE - offset;
2009	if (length > count)
2010	length = count;
2011	p = vmalloc_to_page(addr);
2012	/*
2013	* To do safe access to this _mapped_ area, we need
2014	* lock. But adding lock here means that we need to add
2015	* overhead of vmalloc()/vfree() calles for this _debug_
2016	* interface, rarely used. Instead of that, we'll use
2017	* kmap() and get small overhead in this access function.
2018	*/
2019	if (p) {
2020	/*
2021	* we can expect USER0 is not used (see vread/vwrite's
2022	* function description)
2023	*/
2024	void *map = kmap_atomic(p);
2025	memcpy(map + offset, buf, length);
2026	kunmap_atomic(map);
2027	}
2028	addr += length;
2029	buf += length;
2030	copied += length;
2031	count -= length;
2032	}
2033	return copied;
2034	}
2035
2036	/**
2037	* vread() - read vmalloc area in a safe way.
2038	* @buf: buffer for reading data
2039	* @addr: vm address.
2040	* @count: number of bytes to be read.
2041	*
2042	* Returns # of bytes which addr and buf should be increased.
2043	* (same number to @count). Returns 0 if [addr...addr+count) doesn't
2044	* includes any intersect with alive vmalloc area.
2045	*
2046	* This function checks that addr is a valid vmalloc'ed area, and
2047	* copy data from that area to a given buffer. If the given memory range
2048	* of [addr...addr+count) includes some valid address, data is copied to
2049	* proper area of @buf. If there are memory holes, they'll be zero-filled.
2050	* IOREMAP area is treated as memory hole and no copy is done.
2051	*
2052	* If [addr...addr+count) doesn't includes any intersects with alive
2053	* vm_struct area, returns 0. @buf should be kernel's buffer.
2054	*
2055	* Note: In usual ops, vread() is never necessary because the caller
2056	* should know vmalloc() area is valid and can use memcpy().
2057	* This is for routines which have to access vmalloc area without
2058	* any informaion, as /dev/kmem.
2059	*
2060	*/
2061
2062	long vread(char *buf, char *addr, unsigned long count)
2063	{
2064	struct vmap_area *va;
2065	struct vm_struct *vm;
2066	char *vaddr, *buf_start = buf;
2067	unsigned long buflen = count;
2068	unsigned long n;
2069
2070	/* Don't allow overflow */
2071	if ((unsigned long) addr + count < count)
2072	count = -(unsigned long) addr;
2073
2074	spin_lock(&vmap_area_lock);
2075	list_for_each_entry(va, &vmap_area_list, list) {
2076	if (!count)
2077	break;
2078
2079	if (!(va->flags & VM_VM_AREA))
2080	continue;
2081
2082	vm = va->vm;
2083	vaddr = (char *) vm->addr;
2084	if (addr >= vaddr + get_vm_area_size(vm))
2085	continue;
2086	while (addr < vaddr) {
2087	if (count == 0)
2088	goto finished;
2089	*buf = '\0';
2090	buf++;
2091	addr++;
2092	count--;
2093	}
2094	n = vaddr + get_vm_area_size(vm) - addr;
2095	if (n > count)
2096	n = count;
2097	if (!(vm->flags & VM_IOREMAP))
2098	aligned_vread(buf, addr, n);
2099	else /* IOREMAP area is treated as memory hole */
2100	memset(buf, 0, n);
2101	buf += n;
2102	addr += n;
2103	count -= n;
2104	}
2105	finished:
2106	spin_unlock(&vmap_area_lock);
2107
2108	if (buf == buf_start)
2109	return 0;
2110	/* zero-fill memory holes */
2111	if (buf != buf_start + buflen)
2112	memset(buf, 0, buflen - (buf - buf_start));
2113
2114	return buflen;
2115	}
2116
2117	/**
2118	* vwrite() - write vmalloc area in a safe way.
2119	* @buf: buffer for source data
2120	* @addr: vm address.
2121	* @count: number of bytes to be read.
2122	*
2123	* Returns # of bytes which addr and buf should be incresed.
2124	* (same number to @count).
2125	* If [addr...addr+count) doesn't includes any intersect with valid
2126	* vmalloc area, returns 0.
2127	*
2128	* This function checks that addr is a valid vmalloc'ed area, and
2129	* copy data from a buffer to the given addr. If specified range of
2130	* [addr...addr+count) includes some valid address, data is copied from
2131	* proper area of @buf. If there are memory holes, no copy to hole.
2132	* IOREMAP area is treated as memory hole and no copy is done.
2133	*
2134	* If [addr...addr+count) doesn't includes any intersects with alive
2135	* vm_struct area, returns 0. @buf should be kernel's buffer.
2136	*
2137	* Note: In usual ops, vwrite() is never necessary because the caller
2138	* should know vmalloc() area is valid and can use memcpy().
2139	* This is for routines which have to access vmalloc area without
2140	* any informaion, as /dev/kmem.
2141	*/
2142
2143	long vwrite(char *buf, char *addr, unsigned long count)
2144	{
2145	struct vmap_area *va;
2146	struct vm_struct *vm;
2147	char *vaddr;
2148	unsigned long n, buflen;
2149	int copied = 0;
2150
2151	/* Don't allow overflow */
2152	if ((unsigned long) addr + count < count)
2153	count = -(unsigned long) addr;
2154	buflen = count;
2155
2156	spin_lock(&vmap_area_lock);
2157	list_for_each_entry(va, &vmap_area_list, list) {
2158	if (!count)
2159	break;
2160
2161	if (!(va->flags & VM_VM_AREA))
2162	continue;
2163
2164	vm = va->vm;
2165	vaddr = (char *) vm->addr;
2166	if (addr >= vaddr + get_vm_area_size(vm))
2167	continue;
2168	while (addr < vaddr) {
2169	if (count == 0)
2170	goto finished;
2171	buf++;
2172	addr++;
2173	count--;
2174	}
2175	n = vaddr + get_vm_area_size(vm) - addr;
2176	if (n > count)
2177	n = count;
2178	if (!(vm->flags & VM_IOREMAP)) {
2179	aligned_vwrite(buf, addr, n);
2180	copied++;
2181	}
2182	buf += n;
2183	addr += n;
2184	count -= n;
2185	}
2186	finished:
2187	spin_unlock(&vmap_area_lock);
2188	if (!copied)
2189	return 0;
2190	return buflen;
2191	}
2192
2193	/**
2194	* remap_vmalloc_range_partial - map vmalloc pages to userspace
2195	* @vma: vma to cover
2196	* @uaddr: target user address to start at
2197	* @kaddr: virtual address of vmalloc kernel memory
2198	* @size: size of map area
2199	*
2200	* Returns: 0 for success, -Exxx on failure
2201	*
2202	* This function checks that @kaddr is a valid vmalloc'ed area,
2203	* and that it is big enough to cover the range starting at
2204	* @uaddr in @vma. Will return failure if that criteria isn't
2205	* met.
2206	*
2207	* Similar to remap_pfn_range() (see mm/memory.c)
2208	*/
2209	int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
2210	void *kaddr, unsigned long size)
2211	{
2212	struct vm_struct *area;
2213
2214	size = PAGE_ALIGN(size);
2215
2216	if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
2217	return -EINVAL;
2218
2219	area = find_vm_area(kaddr);
2220	if (!area)
2221	return -EINVAL;
2222
2223	if (!(area->flags & VM_USERMAP))
2224	return -EINVAL;
2225
2226	if (kaddr + size > area->addr + get_vm_area_size(area))
2227	return -EINVAL;
2228
2229	do {
2230	struct page *page = vmalloc_to_page(kaddr);
2231	int ret;
2232
2233	ret = vm_insert_page(vma, uaddr, page);
2234	if (ret)
2235	return ret;
2236
2237	uaddr += PAGE_SIZE;
2238	kaddr += PAGE_SIZE;
2239	size -= PAGE_SIZE;
2240	} while (size > 0);
2241
2242	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
2243
2244	return 0;
2245	}
2246	EXPORT_SYMBOL(remap_vmalloc_range_partial);
2247
2248	/**
2249	* remap_vmalloc_range - map vmalloc pages to userspace
2250	* @vma: vma to cover (map full range of vma)
2251	* @addr: vmalloc memory
2252	* @pgoff: number of pages into addr before first page to map
2253	*
2254	* Returns: 0 for success, -Exxx on failure
2255	*
2256	* This function checks that addr is a valid vmalloc'ed area, and
2257	* that it is big enough to cover the vma. Will return failure if
2258	* that criteria isn't met.
2259	*
2260	* Similar to remap_pfn_range() (see mm/memory.c)
2261	*/
2262	int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
2263	unsigned long pgoff)
2264	{
2265	return remap_vmalloc_range_partial(vma, vma->vm_start,
2266	addr + (pgoff << PAGE_SHIFT),
2267	vma->vm_end - vma->vm_start);
2268	}
2269	EXPORT_SYMBOL(remap_vmalloc_range);
2270
2271	/*
2272	* Implement a stub for vmalloc_sync_all() if the architecture chose not to
2273	* have one.
2274	*/
2275	void __weak vmalloc_sync_all(void)
2276	{
2277	}
2278
2279
2280	static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
2281	{
2282	pte_t ***p = data;
2283
2284	if (p) {
2285	*(*p) = pte;
2286	(*p)++;
2287	}
2288	return 0;
2289	}
2290
2291	/**
2292	* alloc_vm_area - allocate a range of kernel address space
2293	* @size: size of the area
2294	* @ptes: returns the PTEs for the address space
2295	*
2296	* Returns: NULL on failure, vm_struct on success
2297	*
2298	* This function reserves a range of kernel address space, and
2299	* allocates pagetables to map that range. No actual mappings
2300	* are created.
2301	*
2302	* If @ptes is non-NULL, pointers to the PTEs (in init_mm)
2303	* allocated for the VM area are returned.
2304	*/
2305	struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
2306	{
2307	struct vm_struct *area;
2308
2309	area = get_vm_area_caller(size, VM_IOREMAP,
2310	__builtin_return_address(0));
2311	if (area == NULL)
2312	return NULL;
2313
2314	/*
2315	* This ensures that page tables are constructed for this region
2316	* of kernel virtual address space and mapped into init_mm.
2317	*/
2318	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
2319	size, f, ptes ? &ptes : NULL)) {
2320	free_vm_area(area);
2321	return NULL;
2322	}
2323
2324	return area;
2325	}
2326	EXPORT_SYMBOL_GPL(alloc_vm_area);
2327
2328	void free_vm_area(struct vm_struct *area)
2329	{
2330	struct vm_struct *ret;
2331	ret = remove_vm_area(area->addr);
2332	BUG_ON(ret != area);
2333	kfree(area);
2334	}
2335	EXPORT_SYMBOL_GPL(free_vm_area);
2336
2337	#ifdef CONFIG_SMP
2338	static struct vmap_area *node_to_va(struct rb_node *n)
2339	{
2340	return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
2341	}
2342
2343	/**
2344	* pvm_find_next_prev - find the next and prev vmap_area surrounding @end
2345	* @end: target address
2346	* @pnext: out arg for the next vmap_area
2347	* @pprev: out arg for the previous vmap_area
2348	*
2349	* Returns: %true if either or both of next and prev are found,
2350	* %false if no vmap_area exists
2351	*
2352	* Find vmap_areas end addresses of which enclose @end. ie. if not
2353	* NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
2354	*/
2355	static bool pvm_find_next_prev(unsigned long end,
2356	struct vmap_area **pnext,
2357	struct vmap_area **pprev)
2358	{
2359	struct rb_node *n = vmap_area_root.rb_node;
2360	struct vmap_area *va = NULL;
2361
2362	while (n) {
2363	va = rb_entry(n, struct vmap_area, rb_node);
2364	if (end < va->va_end)
2365	n = n->rb_left;
2366	else if (end > va->va_end)
2367	n = n->rb_right;
2368	else
2369	break;
2370	}
2371
2372	if (!va)
2373	return false;
2374
2375	if (va->va_end > end) {
2376	*pnext = va;
2377	*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2378	} else {
2379	*pprev = va;
2380	*pnext = node_to_va(rb_next(&(*pprev)->rb_node));
2381	}
2382	return true;
2383	}
2384
2385	/**
2386	* pvm_determine_end - find the highest aligned address between two vmap_areas
2387	* @pnext: in/out arg for the next vmap_area
2388	* @pprev: in/out arg for the previous vmap_area
2389	* @align: alignment
2390	*
2391	* Returns: determined end address
2392	*
2393	* Find the highest aligned address between *@pnext and *@pprev below
2394	* VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
2395	* down address is between the end addresses of the two vmap_areas.
2396	*
2397	* Please note that the address returned by this function may fall
2398	* inside *@pnext vmap_area. The caller is responsible for checking
2399	* that.
2400	*/
2401	static unsigned long pvm_determine_end(struct vmap_area **pnext,
2402	struct vmap_area **pprev,
2403	unsigned long align)
2404	{
2405	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2406	unsigned long addr;
2407
2408	if (*pnext)
2409	addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
2410	else
2411	addr = vmalloc_end;
2412
2413	while (*pprev && (*pprev)->va_end > addr) {
2414	*pnext = *pprev;
2415	*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2416	}
2417
2418	return addr;
2419	}
2420
2421	/**
2422	* pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
2423	* @offsets: array containing offset of each area
2424	* @sizes: array containing size of each area
2425	* @nr_vms: the number of areas to allocate
2426	* @align: alignment, all entries in @offsets and @sizes must be aligned to this
2427	*
2428	* Returns: kmalloc'd vm_struct pointer array pointing to allocated
2429	* vm_structs on success, %NULL on failure
2430	*
2431	* Percpu allocator wants to use congruent vm areas so that it can
2432	* maintain the offsets among percpu areas. This function allocates
2433	* congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
2434	* be scattered pretty far, distance between two areas easily going up
2435	* to gigabytes. To avoid interacting with regular vmallocs, these
2436	* areas are allocated from top.
2437	*
2438	* Despite its complicated look, this allocator is rather simple. It
2439	* does everything top-down and scans areas from the end looking for
2440	* matching slot. While scanning, if any of the areas overlaps with
2441	* existing vmap_area, the base address is pulled down to fit the
2442	* area. Scanning is repeated till all the areas fit and then all
2443	* necessary data structres are inserted and the result is returned.
2444	*/
2445	struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
2446	const size_t *sizes, int nr_vms,
2447	size_t align)
2448	{
2449	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2450	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2451	struct vmap_area **vas, *prev, *next;
2452	struct vm_struct **vms;
2453	int area, area2, last_area, term_area;
2454	unsigned long base, start, end, last_end;
2455	bool purged = false;
2456
2457	/* verify parameters and allocate data structures */
2458	BUG_ON(offset_in_page(align) || !is_power_of_2(align));
2459	for (last_area = 0, area = 0; area < nr_vms; area++) {
2460	start = offsets[area];
2461	end = start + sizes[area];
2462
2463	/* is everything aligned properly? */
2464	BUG_ON(!IS_ALIGNED(offsets[area], align));
2465	BUG_ON(!IS_ALIGNED(sizes[area], align));
2466
2467	/* detect the area with the highest address */
2468	if (start > offsets[last_area])
2469	last_area = area;
2470
2471	for (area2 = 0; area2 < nr_vms; area2++) {
2472	unsigned long start2 = offsets[area2];
2473	unsigned long end2 = start2 + sizes[area2];
2474
2475	if (area2 == area)
2476	continue;
2477
2478	BUG_ON(start2 >= start && start2 < end);
2479	BUG_ON(end2 <= end && end2 > start);
2480	}
2481	}
2482	last_end = offsets[last_area] + sizes[last_area];
2483
2484	if (vmalloc_end - vmalloc_start < last_end) {
2485	WARN_ON(true);
2486	return NULL;
2487	}
2488
2489	vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
2490	vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
2491	if (!vas || !vms)
2492	goto err_free2;
2493
2494	for (area = 0; area < nr_vms; area++) {
2495	vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
2496	vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
2497	if (!vas[area] || !vms[area])
2498	goto err_free;
2499	}
2500	retry:
2501	spin_lock(&vmap_area_lock);
2502
2503	/* start scanning - we scan from the top, begin with the last area */
2504	area = term_area = last_area;
2505	start = offsets[area];
2506	end = start + sizes[area];
2507
2508	if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
2509	base = vmalloc_end - last_end;
2510	goto found;
2511	}
2512	base = pvm_determine_end(&next, &prev, align) - end;
2513
2514	while (true) {
2515	BUG_ON(next && next->va_end <= base + end);
2516	BUG_ON(prev && prev->va_end > base + end);
2517
2518	/*
2519	* base might have underflowed, add last_end before
2520	* comparing.
2521	*/
2522	if (base + last_end < vmalloc_start + last_end) {
2523	spin_unlock(&vmap_area_lock);
2524	if (!purged) {
2525	purge_vmap_area_lazy();
2526	purged = true;
2527	goto retry;
2528	}
2529	goto err_free;
2530	}
2531
2532	/*
2533	* If next overlaps, move base downwards so that it's
2534	* right below next and then recheck.
2535	*/
2536	if (next && next->va_start < base + end) {
2537	base = pvm_determine_end(&next, &prev, align) - end;
2538	term_area = area;
2539	continue;
2540	}
2541
2542	/*
2543	* If prev overlaps, shift down next and prev and move
2544	* base so that it's right below new next and then
2545	* recheck.
2546	*/
2547	if (prev && prev->va_end > base + start) {
2548	next = prev;
2549	prev = node_to_va(rb_prev(&next->rb_node));
2550	base = pvm_determine_end(&next, &prev, align) - end;
2551	term_area = area;
2552	continue;
2553	}
2554
2555	/*
2556	* This area fits, move on to the previous one. If
2557	* the previous one is the terminal one, we're done.
2558	*/
2559	area = (area + nr_vms - 1) % nr_vms;
2560	if (area == term_area)
2561	break;
2562	start = offsets[area];
2563	end = start + sizes[area];
2564	pvm_find_next_prev(base + end, &next, &prev);
2565	}
2566	found:
2567	/* we've found a fitting base, insert all va's */
2568	for (area = 0; area < nr_vms; area++) {
2569	struct vmap_area *va = vas[area];
2570
2571	va->va_start = base + offsets[area];
2572	va->va_end = va->va_start + sizes[area];
2573	__insert_vmap_area(va);
2574	}
2575
2576	vmap_area_pcpu_hole = base + offsets[last_area];
2577
2578	spin_unlock(&vmap_area_lock);
2579
2580	/* insert all vm's */
2581	for (area = 0; area < nr_vms; area++)
2582	setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
2583	pcpu_get_vm_areas);
2584
2585	kfree(vas);
2586	return vms;
2587
2588	err_free:
2589	for (area = 0; area < nr_vms; area++) {
2590	kfree(vas[area]);
2591	kfree(vms[area]);
2592	}
2593	err_free2:
2594	kfree(vas);
2595	kfree(vms);
2596	return NULL;
2597	}
2598
2599	/**
2600	* pcpu_free_vm_areas - free vmalloc areas for percpu allocator
2601	* @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
2602	* @nr_vms: the number of allocated areas
2603	*
2604	* Free vm_structs and the array allocated by pcpu_get_vm_areas().
2605	*/
2606	void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
2607	{
2608	int i;
2609
2610	for (i = 0; i < nr_vms; i++)
2611	free_vm_area(vms[i]);
2612	kfree(vms);
2613	}
2614	#endif /* CONFIG_SMP */
2615
2616	#ifdef CONFIG_PROC_FS
2617	static void *s_start(struct seq_file *m, loff_t *pos)
2618	__acquires(&vmap_area_lock)
2619	{
2620	loff_t n = *pos;
2621	struct vmap_area *va;
2622
2623	spin_lock(&vmap_area_lock);
2624	va = list_first_entry(&vmap_area_list, typeof(*va), list);
2625	while (n > 0 && &va->list != &vmap_area_list) {
2626	n--;
2627	va = list_next_entry(va, list);
2628	}
2629	if (!n && &va->list != &vmap_area_list)
2630	return va;
2631
2632	return NULL;
2633
2634	}
2635
2636	static void *s_next(struct seq_file *m, void *p, loff_t *pos)
2637	{
2638	struct vmap_area *va = p, *next;
2639
2640	++*pos;
2641	next = list_next_entry(va, list);
2642	if (&next->list != &vmap_area_list)
2643	return next;
2644
2645	return NULL;
2646	}
2647
2648	static void s_stop(struct seq_file *m, void *p)
2649	__releases(&vmap_area_lock)
2650	{
2651	spin_unlock(&vmap_area_lock);
2652	}
2653
2654	static void show_numa_info(struct seq_file *m, struct vm_struct *v)
2655	{
2656	if (IS_ENABLED(CONFIG_NUMA)) {
2657	unsigned int nr, *counters = m->private;
2658
2659	if (!counters)
2660	return;
2661
2662	if (v->flags & VM_UNINITIALIZED)
2663	return;
2664	/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
2665	smp_rmb();
2666
2667	memset(counters, 0, nr_node_ids * sizeof(unsigned int));
2668
2669	for (nr = 0; nr < v->nr_pages; nr++)
2670	counters[page_to_nid(v->pages[nr])]++;
2671
2672	for_each_node_state(nr, N_HIGH_MEMORY)
2673	if (counters[nr])
2674	seq_printf(m, " N%u=%u", nr, counters[nr]);
2675	}
2676	}
2677
2678	static int s_show(struct seq_file *m, void *p)
2679	{
2680	struct vmap_area *va = p;
2681	struct vm_struct *v;
2682
2683	/*
2684	* s_show can encounter race with remove_vm_area, !VM_VM_AREA on
2685	* behalf of vmap area is being tear down or vm_map_ram allocation.
2686	*/
2687	if (!(va->flags & VM_VM_AREA))
2688	return 0;
2689
2690	v = va->vm;
2691
2692	seq_printf(m, "0x%pK-0x%pK %7ld",
2693	v->addr, v->addr + v->size, v->size);
2694
2695	if (v->caller)
2696	seq_printf(m, " %pS", v->caller);
2697
2698	if (v->nr_pages)
2699	seq_printf(m, " pages=%d", v->nr_pages);
2700
2701	if (v->phys_addr)
2702	seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
2703
2704	if (v->flags & VM_IOREMAP)
2705	seq_puts(m, " ioremap");
2706
2707	if (v->flags & VM_ALLOC)
2708	seq_puts(m, " vmalloc");
2709
2710	if (v->flags & VM_MAP)
2711	seq_puts(m, " vmap");
2712
2713	if (v->flags & VM_USERMAP)
2714	seq_puts(m, " user");
2715
2716	if (is_vmalloc_addr(v->pages))
2717	seq_puts(m, " vpages");
2718
2719	show_numa_info(m, v);
2720	seq_putc(m, '\n');
2721	return 0;
2722	}
2723
2724	static const struct seq_operations vmalloc_op = {
2725	.start = s_start,
2726	.next = s_next,
2727	.stop = s_stop,
2728	.show = s_show,
2729	};
2730
2731	static int vmalloc_open(struct inode *inode, struct file *file)
2732	{
2733	if (IS_ENABLED(CONFIG_NUMA))
2734	return seq_open_private(file, &vmalloc_op,
2735	nr_node_ids * sizeof(unsigned int));
2736	else
2737	return seq_open(file, &vmalloc_op);
2738	}
2739
2740	static const struct file_operations proc_vmalloc_operations = {
2741	.open = vmalloc_open,
2742	.read = seq_read,
2743	.llseek = seq_lseek,
2744	.release = seq_release_private,
2745	};
2746
2747	static int __init proc_vmalloc_init(void)
2748	{
2749	proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
2750	return 0;
2751	}
2752	module_init(proc_vmalloc_init);
2753
2754	#endif
2755
2756