path: root/mm/swapfile.c (plain)
blob: 855f62ab8c1b3b3f6c00b0740a3875467938ec3e

1	/*
2	* linux/mm/swapfile.c
3	*
4	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5	* Swap reorganised 29.12.95, Stephen Tweedie
6	*/
7
8	#include <linux/mm.h>
9	#include <linux/hugetlb.h>
10	#include <linux/mman.h>
11	#include <linux/slab.h>
12	#include <linux/kernel_stat.h>
13	#include <linux/swap.h>
14	#include <linux/vmalloc.h>
15	#include <linux/pagemap.h>
16	#include <linux/namei.h>
17	#include <linux/shmem_fs.h>
18	#include <linux/blkdev.h>
19	#include <linux/random.h>
20	#include <linux/writeback.h>
21	#include <linux/proc_fs.h>
22	#include <linux/seq_file.h>
23	#include <linux/init.h>
24	#include <linux/ksm.h>
25	#include <linux/rmap.h>
26	#include <linux/security.h>
27	#include <linux/backing-dev.h>
28	#include <linux/mutex.h>
29	#include <linux/capability.h>
30	#include <linux/syscalls.h>
31	#include <linux/memcontrol.h>
32	#include <linux/poll.h>
33	#include <linux/oom.h>
34	#include <linux/frontswap.h>
35	#include <linux/swapfile.h>
36	#include <linux/export.h>
37
38	#include <asm/pgtable.h>
39	#include <asm/tlbflush.h>
40	#include <linux/swapops.h>
41	#include <linux/swap_cgroup.h>
42
43	static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
44	unsigned char);
45	static void free_swap_count_continuations(struct swap_info_struct *);
46	static sector_t map_swap_entry(swp_entry_t, struct block_device**);
47
48	DEFINE_SPINLOCK(swap_lock);
49	static unsigned int nr_swapfiles;
50	atomic_long_t nr_swap_pages;
51	/*
52	* Some modules use swappable objects and may try to swap them out under
53	* memory pressure (via the shrinker). Before doing so, they may wish to
54	* check to see if any swap space is available.
55	*/
56	EXPORT_SYMBOL_GPL(nr_swap_pages);
57	/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
58	long total_swap_pages;
59	static int least_priority;
60
61	static const char Bad_file[] = "Bad swap file entry ";
62	static const char Unused_file[] = "Unused swap file entry ";
63	static const char Bad_offset[] = "Bad swap offset entry ";
64	static const char Unused_offset[] = "Unused swap offset entry ";
65
66	/*
67	* all active swap_info_structs
68	* protected with swap_lock, and ordered by priority.
69	*/
70	PLIST_HEAD(swap_active_head);
71
72	/*
73	* all available (active, not full) swap_info_structs
74	* protected with swap_avail_lock, ordered by priority.
75	* This is used by get_swap_page() instead of swap_active_head
76	* because swap_active_head includes all swap_info_structs,
77	* but get_swap_page() doesn't need to look at full ones.
78	* This uses its own lock instead of swap_lock because when a
79	* swap_info_struct changes between not-full/full, it needs to
80	* add/remove itself to/from this list, but the swap_info_struct->lock
81	* is held and the locking order requires swap_lock to be taken
82	* before any swap_info_struct->lock.
83	*/
84	static PLIST_HEAD(swap_avail_head);
85	static DEFINE_SPINLOCK(swap_avail_lock);
86
87	struct swap_info_struct *swap_info[MAX_SWAPFILES];
88
89	static DEFINE_MUTEX(swapon_mutex);
90
91	static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
92	/* Activity counter to indicate that a swapon or swapoff has occurred */
93	static atomic_t proc_poll_event = ATOMIC_INIT(0);
94
95	static inline unsigned char swap_count(unsigned char ent)
96	{
97	return ent & ~SWAP_HAS_CACHE; /* may include SWAP_HAS_CONT flag */
98	}
99
100	/* returns 1 if swap entry is freed */
101	static int
102	__try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
103	{
104	swp_entry_t entry = swp_entry(si->type, offset);
105	struct page *page;
106	int ret = 0;
107
108	page = find_get_page(swap_address_space(entry), swp_offset(entry));
109	if (!page)
110	return 0;
111	/*
112	* This function is called from scan_swap_map() and it's called
113	* by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
114	* We have to use trylock for avoiding deadlock. This is a special
115	* case and you should use try_to_free_swap() with explicit lock_page()
116	* in usual operations.
117	*/
118	if (trylock_page(page)) {
119	ret = try_to_free_swap(page);
120	unlock_page(page);
121	}
122	put_page(page);
123	return ret;
124	}
125
126	/*
127	* swapon tell device that all the old swap contents can be discarded,
128	* to allow the swap device to optimize its wear-levelling.
129	*/
130	static int discard_swap(struct swap_info_struct *si)
131	{
132	struct swap_extent *se;
133	sector_t start_block;
134	sector_t nr_blocks;
135	int err = 0;
136
137	/* Do not discard the swap header page! */
138	se = &si->first_swap_extent;
139	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
140	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
141	if (nr_blocks) {
142	err = blkdev_issue_discard(si->bdev, start_block,
143	nr_blocks, GFP_KERNEL, 0);
144	if (err)
145	return err;
146	cond_resched();
147	}
148
149	list_for_each_entry(se, &si->first_swap_extent.list, list) {
150	start_block = se->start_block << (PAGE_SHIFT - 9);
151	nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
152
153	err = blkdev_issue_discard(si->bdev, start_block,
154	nr_blocks, GFP_KERNEL, 0);
155	if (err)
156	break;
157
158	cond_resched();
159	}
160	return err; /* That will often be -EOPNOTSUPP */
161	}
162
163	/*
164	* swap allocation tell device that a cluster of swap can now be discarded,
165	* to allow the swap device to optimize its wear-levelling.
166	*/
167	static void discard_swap_cluster(struct swap_info_struct *si,
168	pgoff_t start_page, pgoff_t nr_pages)
169	{
170	struct swap_extent *se = si->curr_swap_extent;
171	int found_extent = 0;
172
173	while (nr_pages) {
174	if (se->start_page <= start_page &&
175	start_page < se->start_page + se->nr_pages) {
176	pgoff_t offset = start_page - se->start_page;
177	sector_t start_block = se->start_block + offset;
178	sector_t nr_blocks = se->nr_pages - offset;
179
180	if (nr_blocks > nr_pages)
181	nr_blocks = nr_pages;
182	start_page += nr_blocks;
183	nr_pages -= nr_blocks;
184
185	if (!found_extent++)
186	si->curr_swap_extent = se;
187
188	start_block <<= PAGE_SHIFT - 9;
189	nr_blocks <<= PAGE_SHIFT - 9;
190	if (blkdev_issue_discard(si->bdev, start_block,
191	nr_blocks, GFP_NOIO, 0))
192	break;
193	}
194
195	se = list_next_entry(se, list);
196	}
197	}
198
199	#define SWAPFILE_CLUSTER 256
200	#define LATENCY_LIMIT 256
201
202	static inline void cluster_set_flag(struct swap_cluster_info *info,
203	unsigned int flag)
204	{
205	info->flags = flag;
206	}
207
208	static inline unsigned int cluster_count(struct swap_cluster_info *info)
209	{
210	return info->data;
211	}
212
213	static inline void cluster_set_count(struct swap_cluster_info *info,
214	unsigned int c)
215	{
216	info->data = c;
217	}
218
219	static inline void cluster_set_count_flag(struct swap_cluster_info *info,
220	unsigned int c, unsigned int f)
221	{
222	info->flags = f;
223	info->data = c;
224	}
225
226	static inline unsigned int cluster_next(struct swap_cluster_info *info)
227	{
228	return info->data;
229	}
230
231	static inline void cluster_set_next(struct swap_cluster_info *info,
232	unsigned int n)
233	{
234	info->data = n;
235	}
236
237	static inline void cluster_set_next_flag(struct swap_cluster_info *info,
238	unsigned int n, unsigned int f)
239	{
240	info->flags = f;
241	info->data = n;
242	}
243
244	static inline bool cluster_is_free(struct swap_cluster_info *info)
245	{
246	return info->flags & CLUSTER_FLAG_FREE;
247	}
248
249	static inline bool cluster_is_null(struct swap_cluster_info *info)
250	{
251	return info->flags & CLUSTER_FLAG_NEXT_NULL;
252	}
253
254	static inline void cluster_set_null(struct swap_cluster_info *info)
255	{
256	info->flags = CLUSTER_FLAG_NEXT_NULL;
257	info->data = 0;
258	}
259
260	static inline bool cluster_list_empty(struct swap_cluster_list *list)
261	{
262	return cluster_is_null(&list->head);
263	}
264
265	static inline unsigned int cluster_list_first(struct swap_cluster_list *list)
266	{
267	return cluster_next(&list->head);
268	}
269
270	static void cluster_list_init(struct swap_cluster_list *list)
271	{
272	cluster_set_null(&list->head);
273	cluster_set_null(&list->tail);
274	}
275
276	static void cluster_list_add_tail(struct swap_cluster_list *list,
277	struct swap_cluster_info *ci,
278	unsigned int idx)
279	{
280	if (cluster_list_empty(list)) {
281	cluster_set_next_flag(&list->head, idx, 0);
282	cluster_set_next_flag(&list->tail, idx, 0);
283	} else {
284	unsigned int tail = cluster_next(&list->tail);
285
286	cluster_set_next(&ci[tail], idx);
287	cluster_set_next_flag(&list->tail, idx, 0);
288	}
289	}
290
291	static unsigned int cluster_list_del_first(struct swap_cluster_list *list,
292	struct swap_cluster_info *ci)
293	{
294	unsigned int idx;
295
296	idx = cluster_next(&list->head);
297	if (cluster_next(&list->tail) == idx) {
298	cluster_set_null(&list->head);
299	cluster_set_null(&list->tail);
300	} else
301	cluster_set_next_flag(&list->head,
302	cluster_next(&ci[idx]), 0);
303
304	return idx;
305	}
306
307	/* Add a cluster to discard list and schedule it to do discard */
308	static void swap_cluster_schedule_discard(struct swap_info_struct *si,
309	unsigned int idx)
310	{
311	/*
312	* If scan_swap_map() can't find a free cluster, it will check
313	* si->swap_map directly. To make sure the discarding cluster isn't
314	* taken by scan_swap_map(), mark the swap entries bad (occupied). It
315	* will be cleared after discard
316	*/
317	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
318	SWAP_MAP_BAD, SWAPFILE_CLUSTER);
319
320	cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx);
321
322	schedule_work(&si->discard_work);
323	}
324
325	/*
326	* Doing discard actually. After a cluster discard is finished, the cluster
327	* will be added to free cluster list. caller should hold si->lock.
328	*/
329	static void swap_do_scheduled_discard(struct swap_info_struct *si)
330	{
331	struct swap_cluster_info *info;
332	unsigned int idx;
333
334	info = si->cluster_info;
335
336	while (!cluster_list_empty(&si->discard_clusters)) {
337	idx = cluster_list_del_first(&si->discard_clusters, info);
338	spin_unlock(&si->lock);
339
340	discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
341	SWAPFILE_CLUSTER);
342
343	spin_lock(&si->lock);
344	cluster_set_flag(&info[idx], CLUSTER_FLAG_FREE);
345	cluster_list_add_tail(&si->free_clusters, info, idx);
346	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
347	0, SWAPFILE_CLUSTER);
348	}
349	}
350
351	static void swap_discard_work(struct work_struct *work)
352	{
353	struct swap_info_struct *si;
354
355	si = container_of(work, struct swap_info_struct, discard_work);
356
357	spin_lock(&si->lock);
358	swap_do_scheduled_discard(si);
359	spin_unlock(&si->lock);
360	}
361
362	/*
363	* The cluster corresponding to page_nr will be used. The cluster will be
364	* removed from free cluster list and its usage counter will be increased.
365	*/
366	static void inc_cluster_info_page(struct swap_info_struct *p,
367	struct swap_cluster_info *cluster_info, unsigned long page_nr)
368	{
369	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
370
371	if (!cluster_info)
372	return;
373	if (cluster_is_free(&cluster_info[idx])) {
374	VM_BUG_ON(cluster_list_first(&p->free_clusters) != idx);
375	cluster_list_del_first(&p->free_clusters, cluster_info);
376	cluster_set_count_flag(&cluster_info[idx], 0, 0);
377	}
378
379	VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
380	cluster_set_count(&cluster_info[idx],
381	cluster_count(&cluster_info[idx]) + 1);
382	}
383
384	/*
385	* The cluster corresponding to page_nr decreases one usage. If the usage
386	* counter becomes 0, which means no page in the cluster is in using, we can
387	* optionally discard the cluster and add it to free cluster list.
388	*/
389	static void dec_cluster_info_page(struct swap_info_struct *p,
390	struct swap_cluster_info *cluster_info, unsigned long page_nr)
391	{
392	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
393
394	if (!cluster_info)
395	return;
396
397	VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
398	cluster_set_count(&cluster_info[idx],
399	cluster_count(&cluster_info[idx]) - 1);
400
401	if (cluster_count(&cluster_info[idx]) == 0) {
402	/*
403	* If the swap is discardable, prepare discard the cluster
404	* instead of free it immediately. The cluster will be freed
405	* after discard.
406	*/
407	if ((p->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
408	(SWP_WRITEOK | SWP_PAGE_DISCARD)) {
409	swap_cluster_schedule_discard(p, idx);
410	return;
411	}
412
413	cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
414	cluster_list_add_tail(&p->free_clusters, cluster_info, idx);
415	}
416	}
417
418	/*
419	* It's possible scan_swap_map() uses a free cluster in the middle of free
420	* cluster list. Avoiding such abuse to avoid list corruption.
421	*/
422	static bool
423	scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
424	unsigned long offset)
425	{
426	struct percpu_cluster *percpu_cluster;
427	bool conflict;
428
429	offset /= SWAPFILE_CLUSTER;
430	conflict = !cluster_list_empty(&si->free_clusters) &&
431	offset != cluster_list_first(&si->free_clusters) &&
432	cluster_is_free(&si->cluster_info[offset]);
433
434	if (!conflict)
435	return false;
436
437	percpu_cluster = this_cpu_ptr(si->percpu_cluster);
438	cluster_set_null(&percpu_cluster->index);
439	return true;
440	}
441
442	/*
443	* Try to get a swap entry from current cpu's swap entry pool (a cluster). This
444	* might involve allocating a new cluster for current CPU too.
445	*/
446	static void scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
447	unsigned long *offset, unsigned long *scan_base)
448	{
449	struct percpu_cluster *cluster;
450	bool found_free;
451	unsigned long tmp;
452
453	new_cluster:
454	cluster = this_cpu_ptr(si->percpu_cluster);
455	if (cluster_is_null(&cluster->index)) {
456	if (!cluster_list_empty(&si->free_clusters)) {
457	cluster->index = si->free_clusters.head;
458	cluster->next = cluster_next(&cluster->index) *
459	SWAPFILE_CLUSTER;
460	} else if (!cluster_list_empty(&si->discard_clusters)) {
461	/*
462	* we don't have free cluster but have some clusters in
463	* discarding, do discard now and reclaim them
464	*/
465	swap_do_scheduled_discard(si);
466	*scan_base = *offset = si->cluster_next;
467	goto new_cluster;
468	} else
469	return;
470	}
471
472	found_free = false;
473
474	/*
475	* Other CPUs can use our cluster if they can't find a free cluster,
476	* check if there is still free entry in the cluster
477	*/
478	tmp = cluster->next;
479	while (tmp < si->max && tmp < (cluster_next(&cluster->index) + 1) *
480	SWAPFILE_CLUSTER) {
481	if (!si->swap_map[tmp]) {
482	found_free = true;
483	break;
484	}
485	tmp++;
486	}
487	if (!found_free) {
488	cluster_set_null(&cluster->index);
489	goto new_cluster;
490	}
491	cluster->next = tmp + 1;
492	*offset = tmp;
493	*scan_base = tmp;
494	}
495
496	static unsigned long scan_swap_map(struct swap_info_struct *si,
497	unsigned char usage)
498	{
499	unsigned long offset;
500	unsigned long scan_base;
501	unsigned long last_in_cluster = 0;
502	int latency_ration = LATENCY_LIMIT;
503
504	/*
505	* We try to cluster swap pages by allocating them sequentially
506	* in swap. Once we've allocated SWAPFILE_CLUSTER pages this
507	* way, however, we resort to first-free allocation, starting
508	* a new cluster. This prevents us from scattering swap pages
509	* all over the entire swap partition, so that we reduce
510	* overall disk seek times between swap pages. -- sct
511	* But we do now try to find an empty cluster. -Andrea
512	* And we let swap pages go all over an SSD partition. Hugh
513	*/
514
515	si->flags += SWP_SCANNING;
516	scan_base = offset = si->cluster_next;
517
518	/* SSD algorithm */
519	if (si->cluster_info) {
520	scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
521	goto checks;
522	}
523
524	if (unlikely(!si->cluster_nr--)) {
525	if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
526	si->cluster_nr = SWAPFILE_CLUSTER - 1;
527	goto checks;
528	}
529
530	spin_unlock(&si->lock);
531
532	/*
533	* If seek is expensive, start searching for new cluster from
534	* start of partition, to minimize the span of allocated swap.
535	* If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
536	* case, just handled by scan_swap_map_try_ssd_cluster() above.
537	*/
538	scan_base = offset = si->lowest_bit;
539	last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
540
541	/* Locate the first empty (unaligned) cluster */
542	for (; last_in_cluster <= si->highest_bit; offset++) {
543	if (si->swap_map[offset])
544	last_in_cluster = offset + SWAPFILE_CLUSTER;
545	else if (offset == last_in_cluster) {
546	spin_lock(&si->lock);
547	offset -= SWAPFILE_CLUSTER - 1;
548	si->cluster_next = offset;
549	si->cluster_nr = SWAPFILE_CLUSTER - 1;
550	goto checks;
551	}
552	if (unlikely(--latency_ration < 0)) {
553	cond_resched();
554	latency_ration = LATENCY_LIMIT;
555	}
556	}
557
558	offset = scan_base;
559	spin_lock(&si->lock);
560	si->cluster_nr = SWAPFILE_CLUSTER - 1;
561	}
562
563	checks:
564	if (si->cluster_info) {
565	while (scan_swap_map_ssd_cluster_conflict(si, offset))
566	scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
567	}
568	if (!(si->flags & SWP_WRITEOK))
569	goto no_page;
570	if (!si->highest_bit)
571	goto no_page;
572	if (offset > si->highest_bit)
573	scan_base = offset = si->lowest_bit;
574
575	/* reuse swap entry of cache-only swap if not busy. */
576	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
577	int swap_was_freed;
578	spin_unlock(&si->lock);
579	swap_was_freed = __try_to_reclaim_swap(si, offset);
580	spin_lock(&si->lock);
581	/* entry was freed successfully, try to use this again */
582	if (swap_was_freed)
583	goto checks;
584	goto scan; /* check next one */
585	}
586
587	if (si->swap_map[offset])
588	goto scan;
589
590	if (offset == si->lowest_bit)
591	si->lowest_bit++;
592	if (offset == si->highest_bit)
593	si->highest_bit--;
594	si->inuse_pages++;
595	if (si->inuse_pages == si->pages) {
596	si->lowest_bit = si->max;
597	si->highest_bit = 0;
598	spin_lock(&swap_avail_lock);
599	plist_del(&si->avail_list, &swap_avail_head);
600	spin_unlock(&swap_avail_lock);
601	}
602	si->swap_map[offset] = usage;
603	inc_cluster_info_page(si, si->cluster_info, offset);
604	si->cluster_next = offset + 1;
605	si->flags -= SWP_SCANNING;
606
607	return offset;
608
609	scan:
610	spin_unlock(&si->lock);
611	while (++offset <= si->highest_bit) {
612	if (!si->swap_map[offset]) {
613	spin_lock(&si->lock);
614	goto checks;
615	}
616	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
617	spin_lock(&si->lock);
618	goto checks;
619	}
620	if (unlikely(--latency_ration < 0)) {
621	cond_resched();
622	latency_ration = LATENCY_LIMIT;
623	}
624	}
625	offset = si->lowest_bit;
626	while (offset < scan_base) {
627	if (!si->swap_map[offset]) {
628	spin_lock(&si->lock);
629	goto checks;
630	}
631	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
632	spin_lock(&si->lock);
633	goto checks;
634	}
635	if (unlikely(--latency_ration < 0)) {
636	cond_resched();
637	latency_ration = LATENCY_LIMIT;
638	}
639	offset++;
640	}
641	spin_lock(&si->lock);
642
643	no_page:
644	si->flags -= SWP_SCANNING;
645	return 0;
646	}
647
648	swp_entry_t get_swap_page(void)
649	{
650	struct swap_info_struct *si, *next;
651	pgoff_t offset;
652
653	if (atomic_long_read(&nr_swap_pages) <= 0)
654	goto noswap;
655	atomic_long_dec(&nr_swap_pages);
656
657	spin_lock(&swap_avail_lock);
658
659	start_over:
660	plist_for_each_entry_safe(si, next, &swap_avail_head, avail_list) {
661	/* requeue si to after same-priority siblings */
662	plist_requeue(&si->avail_list, &swap_avail_head);
663	spin_unlock(&swap_avail_lock);
664	spin_lock(&si->lock);
665	if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
666	spin_lock(&swap_avail_lock);
667	if (plist_node_empty(&si->avail_list)) {
668	spin_unlock(&si->lock);
669	goto nextsi;
670	}
671	WARN(!si->highest_bit,
672	"swap_info %d in list but !highest_bit\n",
673	si->type);
674	WARN(!(si->flags & SWP_WRITEOK),
675	"swap_info %d in list but !SWP_WRITEOK\n",
676	si->type);
677	plist_del(&si->avail_list, &swap_avail_head);
678	spin_unlock(&si->lock);
679	goto nextsi;
680	}
681
682	/* This is called for allocating swap entry for cache */
683	offset = scan_swap_map(si, SWAP_HAS_CACHE);
684	spin_unlock(&si->lock);
685	if (offset)
686	return swp_entry(si->type, offset);
687	pr_debug("scan_swap_map of si %d failed to find offset\n",
688	si->type);
689	spin_lock(&swap_avail_lock);
690	nextsi:
691	/*
692	* if we got here, it's likely that si was almost full before,
693	* and since scan_swap_map() can drop the si->lock, multiple
694	* callers probably all tried to get a page from the same si
695	* and it filled up before we could get one; or, the si filled
696	* up between us dropping swap_avail_lock and taking si->lock.
697	* Since we dropped the swap_avail_lock, the swap_avail_head
698	* list may have been modified; so if next is still in the
699	* swap_avail_head list then try it, otherwise start over.
700	*/
701	if (plist_node_empty(&next->avail_list))
702	goto start_over;
703	}
704
705	spin_unlock(&swap_avail_lock);
706
707	atomic_long_inc(&nr_swap_pages);
708	noswap:
709	return (swp_entry_t) {0};
710	}
711
712	/* The only caller of this function is now suspend routine */
713	swp_entry_t get_swap_page_of_type(int type)
714	{
715	struct swap_info_struct *si;
716	pgoff_t offset;
717
718	si = swap_info[type];
719	spin_lock(&si->lock);
720	if (si && (si->flags & SWP_WRITEOK)) {
721	atomic_long_dec(&nr_swap_pages);
722	/* This is called for allocating swap entry, not cache */
723	offset = scan_swap_map(si, 1);
724	if (offset) {
725	spin_unlock(&si->lock);
726	return swp_entry(type, offset);
727	}
728	atomic_long_inc(&nr_swap_pages);
729	}
730	spin_unlock(&si->lock);
731	return (swp_entry_t) {0};
732	}
733
734	static struct swap_info_struct *swap_info_get(swp_entry_t entry)
735	{
736	struct swap_info_struct *p;
737	unsigned long offset, type;
738
739	if (!entry.val)
740	goto out;
741	type = swp_type(entry);
742	if (type >= nr_swapfiles)
743	goto bad_nofile;
744	p = swap_info[type];
745	if (!(p->flags & SWP_USED))
746	goto bad_device;
747	offset = swp_offset(entry);
748	if (offset >= p->max)
749	goto bad_offset;
750	if (!p->swap_map[offset])
751	goto bad_free;
752	spin_lock(&p->lock);
753	return p;
754
755	bad_free:
756	pr_err("swap_free: %s%08lx\n", Unused_offset, entry.val);
757	goto out;
758	bad_offset:
759	pr_err("swap_free: %s%08lx\n", Bad_offset, entry.val);
760	goto out;
761	bad_device:
762	pr_err("swap_free: %s%08lx\n", Unused_file, entry.val);
763	goto out;
764	bad_nofile:
765	pr_err("swap_free: %s%08lx\n", Bad_file, entry.val);
766	out:
767	return NULL;
768	}
769
770	static unsigned char swap_entry_free(struct swap_info_struct *p,
771	swp_entry_t entry, unsigned char usage)
772	{
773	unsigned long offset = swp_offset(entry);
774	unsigned char count;
775	unsigned char has_cache;
776
777	count = p->swap_map[offset];
778	has_cache = count & SWAP_HAS_CACHE;
779	count &= ~SWAP_HAS_CACHE;
780
781	if (usage == SWAP_HAS_CACHE) {
782	VM_BUG_ON(!has_cache);
783	has_cache = 0;
784	} else if (count == SWAP_MAP_SHMEM) {
785	/*
786	* Or we could insist on shmem.c using a special
787	* swap_shmem_free() and free_shmem_swap_and_cache()...
788	*/
789	count = 0;
790	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
791	if (count == COUNT_CONTINUED) {
792	if (swap_count_continued(p, offset, count))
793	count = SWAP_MAP_MAX | COUNT_CONTINUED;
794	else
795	count = SWAP_MAP_MAX;
796	} else
797	count--;
798	}
799
800	usage = count | has_cache;
801	p->swap_map[offset] = usage;
802
803	/* free if no reference */
804	if (!usage) {
805	mem_cgroup_uncharge_swap(entry);
806	dec_cluster_info_page(p, p->cluster_info, offset);
807	if (offset < p->lowest_bit)
808	p->lowest_bit = offset;
809	if (offset > p->highest_bit) {
810	bool was_full = !p->highest_bit;
811	p->highest_bit = offset;
812	if (was_full && (p->flags & SWP_WRITEOK)) {
813	spin_lock(&swap_avail_lock);
814	WARN_ON(!plist_node_empty(&p->avail_list));
815	if (plist_node_empty(&p->avail_list))
816	plist_add(&p->avail_list,
817	&swap_avail_head);
818	spin_unlock(&swap_avail_lock);
819	}
820	}
821	atomic_long_inc(&nr_swap_pages);
822	p->inuse_pages--;
823	frontswap_invalidate_page(p->type, offset);
824	if (p->flags & SWP_BLKDEV) {
825	struct gendisk *disk = p->bdev->bd_disk;
826	if (disk->fops->swap_slot_free_notify)
827	disk->fops->swap_slot_free_notify(p->bdev,
828	offset);
829	}
830	}
831
832	return usage;
833	}
834
835	/*
836	* Caller has made sure that the swap device corresponding to entry
837	* is still around or has not been recycled.
838	*/
839	void swap_free(swp_entry_t entry)
840	{
841	struct swap_info_struct *p;
842
843	p = swap_info_get(entry);
844	if (p) {
845	swap_entry_free(p, entry, 1);
846	spin_unlock(&p->lock);
847	}
848	}
849
850	/*
851	* Called after dropping swapcache to decrease refcnt to swap entries.
852	*/
853	void swapcache_free(swp_entry_t entry)
854	{
855	struct swap_info_struct *p;
856
857	p = swap_info_get(entry);
858	if (p) {
859	swap_entry_free(p, entry, SWAP_HAS_CACHE);
860	spin_unlock(&p->lock);
861	}
862	}
863
864	/*
865	* How many references to page are currently swapped out?
866	* This does not give an exact answer when swap count is continued,
867	* but does include the high COUNT_CONTINUED flag to allow for that.
868	*/
869	int page_swapcount(struct page *page)
870	{
871	int count = 0;
872	struct swap_info_struct *p;
873	swp_entry_t entry;
874
875	entry.val = page_private(page);
876	p = swap_info_get(entry);
877	if (p) {
878	count = swap_count(p->swap_map[swp_offset(entry)]);
879	spin_unlock(&p->lock);
880	}
881	return count;
882	}
883
884	/*
885	* How many references to @entry are currently swapped out?
886	* This considers COUNT_CONTINUED so it returns exact answer.
887	*/
888	int swp_swapcount(swp_entry_t entry)
889	{
890	int count, tmp_count, n;
891	struct swap_info_struct *p;
892	struct page *page;
893	pgoff_t offset;
894	unsigned char *map;
895
896	p = swap_info_get(entry);
897	if (!p)
898	return 0;
899
900	count = swap_count(p->swap_map[swp_offset(entry)]);
901	if (!(count & COUNT_CONTINUED))
902	goto out;
903
904	count &= ~COUNT_CONTINUED;
905	n = SWAP_MAP_MAX + 1;
906
907	offset = swp_offset(entry);
908	page = vmalloc_to_page(p->swap_map + offset);
909	offset &= ~PAGE_MASK;
910	VM_BUG_ON(page_private(page) != SWP_CONTINUED);
911
912	do {
913	page = list_next_entry(page, lru);
914	map = kmap_atomic(page);
915	tmp_count = map[offset];
916	kunmap_atomic(map);
917
918	count += (tmp_count & ~COUNT_CONTINUED) * n;
919	n *= (SWAP_CONT_MAX + 1);
920	} while (tmp_count & COUNT_CONTINUED);
921	out:
922	spin_unlock(&p->lock);
923	return count;
924	}
925
926	/*
927	* We can write to an anon page without COW if there are no other references
928	* to it. And as a side-effect, free up its swap: because the old content
929	* on disk will never be read, and seeking back there to write new content
930	* later would only waste time away from clustering.
931	*
932	* NOTE: total_mapcount should not be relied upon by the caller if
933	* reuse_swap_page() returns false, but it may be always overwritten
934	* (see the other implementation for CONFIG_SWAP=n).
935	*/
936	bool reuse_swap_page(struct page *page, int *total_mapcount)
937	{
938	int count;
939
940	VM_BUG_ON_PAGE(!PageLocked(page), page);
941	if (unlikely(PageKsm(page)))
942	return false;
943	count = page_trans_huge_mapcount(page, total_mapcount);
944	if (count <= 1 && PageSwapCache(page)) {
945	count += page_swapcount(page);
946	if (count != 1)
947	goto out;
948	if (!PageWriteback(page)) {
949	delete_from_swap_cache(page);
950	SetPageDirty(page);
951	} else {
952	swp_entry_t entry;
953	struct swap_info_struct *p;
954
955	entry.val = page_private(page);
956	p = swap_info_get(entry);
957	if (p->flags & SWP_STABLE_WRITES) {
958	spin_unlock(&p->lock);
959	return false;
960	}
961	spin_unlock(&p->lock);
962	}
963	}
964	out:
965	return count <= 1;
966	}
967
968	/*
969	* If swap is getting full, or if there are no more mappings of this page,
970	* then try_to_free_swap is called to free its swap space.
971	*/
972	int try_to_free_swap(struct page *page)
973	{
974	VM_BUG_ON_PAGE(!PageLocked(page), page);
975
976	if (!PageSwapCache(page))
977	return 0;
978	if (PageWriteback(page))
979	return 0;
980	if (page_swapcount(page))
981	return 0;
982
983	/*
984	* Once hibernation has begun to create its image of memory,
985	* there's a danger that one of the calls to try_to_free_swap()
986	* - most probably a call from __try_to_reclaim_swap() while
987	* hibernation is allocating its own swap pages for the image,
988	* but conceivably even a call from memory reclaim - will free
989	* the swap from a page which has already been recorded in the
990	* image as a clean swapcache page, and then reuse its swap for
991	* another page of the image. On waking from hibernation, the
992	* original page might be freed under memory pressure, then
993	* later read back in from swap, now with the wrong data.
994	*
995	* Hibernation suspends storage while it is writing the image
996	* to disk so check that here.
997	*/
998	if (pm_suspended_storage())
999	return 0;
1000
1001	delete_from_swap_cache(page);
1002	SetPageDirty(page);
1003	return 1;
1004	}
1005
1006	/*
1007	* Free the swap entry like above, but also try to
1008	* free the page cache entry if it is the last user.
1009	*/
1010	int free_swap_and_cache(swp_entry_t entry)
1011	{
1012	struct swap_info_struct *p;
1013	struct page *page = NULL;
1014
1015	if (non_swap_entry(entry))
1016	return 1;
1017
1018	p = swap_info_get(entry);
1019	if (p) {
1020	if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
1021	page = find_get_page(swap_address_space(entry),
1022	swp_offset(entry));
1023	if (page && !trylock_page(page)) {
1024	put_page(page);
1025	page = NULL;
1026	}
1027	}
1028	spin_unlock(&p->lock);
1029	}
1030	if (page) {
1031	/*
1032	* Not mapped elsewhere, or swap space full? Free it!
1033	* Also recheck PageSwapCache now page is locked (above).
1034	*/
1035	if (PageSwapCache(page) && !PageWriteback(page) &&
1036	(!page_mapped(page) || mem_cgroup_swap_full(page))) {
1037	delete_from_swap_cache(page);
1038	SetPageDirty(page);
1039	}
1040	unlock_page(page);
1041	put_page(page);
1042	}
1043	return p != NULL;
1044	}
1045
1046	#ifdef CONFIG_HIBERNATION
1047	/*
1048	* Find the swap type that corresponds to given device (if any).
1049	*
1050	* @offset - number of the PAGE_SIZE-sized block of the device, starting
1051	* from 0, in which the swap header is expected to be located.
1052	*
1053	* This is needed for the suspend to disk (aka swsusp).
1054	*/
1055	int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1056	{
1057	struct block_device *bdev = NULL;
1058	int type;
1059
1060	if (device)
1061	bdev = bdget(device);
1062
1063	spin_lock(&swap_lock);
1064	for (type = 0; type < nr_swapfiles; type++) {
1065	struct swap_info_struct *sis = swap_info[type];
1066
1067	if (!(sis->flags & SWP_WRITEOK))
1068	continue;
1069
1070	if (!bdev) {
1071	if (bdev_p)
1072	*bdev_p = bdgrab(sis->bdev);
1073
1074	spin_unlock(&swap_lock);
1075	return type;
1076	}
1077	if (bdev == sis->bdev) {
1078	struct swap_extent *se = &sis->first_swap_extent;
1079
1080	if (se->start_block == offset) {
1081	if (bdev_p)
1082	*bdev_p = bdgrab(sis->bdev);
1083
1084	spin_unlock(&swap_lock);
1085	bdput(bdev);
1086	return type;
1087	}
1088	}
1089	}
1090	spin_unlock(&swap_lock);
1091	if (bdev)
1092	bdput(bdev);
1093
1094	return -ENODEV;
1095	}
1096
1097	/*
1098	* Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1099	* corresponding to given index in swap_info (swap type).
1100	*/
1101	sector_t swapdev_block(int type, pgoff_t offset)
1102	{
1103	struct block_device *bdev;
1104
1105	if ((unsigned int)type >= nr_swapfiles)
1106	return 0;
1107	if (!(swap_info[type]->flags & SWP_WRITEOK))
1108	return 0;
1109	return map_swap_entry(swp_entry(type, offset), &bdev);
1110	}
1111
1112	/*
1113	* Return either the total number of swap pages of given type, or the number
1114	* of free pages of that type (depending on @free)
1115	*
1116	* This is needed for software suspend
1117	*/
1118	unsigned int count_swap_pages(int type, int free)
1119	{
1120	unsigned int n = 0;
1121
1122	spin_lock(&swap_lock);
1123	if ((unsigned int)type < nr_swapfiles) {
1124	struct swap_info_struct *sis = swap_info[type];
1125
1126	spin_lock(&sis->lock);
1127	if (sis->flags & SWP_WRITEOK) {
1128	n = sis->pages;
1129	if (free)
1130	n -= sis->inuse_pages;
1131	}
1132	spin_unlock(&sis->lock);
1133	}
1134	spin_unlock(&swap_lock);
1135	return n;
1136	}
1137	#endif /* CONFIG_HIBERNATION */
1138
1139	static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
1140	{
1141	return pte_same(pte_swp_clear_soft_dirty(pte), swp_pte);
1142	}
1143
1144	/*
1145	* No need to decide whether this PTE shares the swap entry with others,
1146	* just let do_wp_page work it out if a write is requested later - to
1147	* force COW, vm_page_prot omits write permission from any private vma.
1148	*/
1149	static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1150	unsigned long addr, swp_entry_t entry, struct page *page)
1151	{
1152	struct page *swapcache;
1153	struct mem_cgroup *memcg;
1154	spinlock_t *ptl;
1155	pte_t *pte;
1156	int ret = 1;
1157
1158	swapcache = page;
1159	page = ksm_might_need_to_copy(page, vma, addr);
1160	if (unlikely(!page))
1161	return -ENOMEM;
1162
1163	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
1164	&memcg, false)) {
1165	ret = -ENOMEM;
1166	goto out_nolock;
1167	}
1168
1169	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1170	if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) {
1171	mem_cgroup_cancel_charge(page, memcg, false);
1172	ret = 0;
1173	goto out;
1174	}
1175
1176	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1177	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1178	get_page(page);
1179	set_pte_at(vma->vm_mm, addr, pte,
1180	pte_mkold(mk_pte(page, vma->vm_page_prot)));
1181	if (page == swapcache) {
1182	page_add_anon_rmap(page, vma, addr, false);
1183	mem_cgroup_commit_charge(page, memcg, true, false);
1184	} else { /* ksm created a completely new copy */
1185	page_add_new_anon_rmap(page, vma, addr, false);
1186	mem_cgroup_commit_charge(page, memcg, false, false);
1187	lru_cache_add_active_or_unevictable(page, vma);
1188	}
1189	swap_free(entry);
1190	/*
1191	* Move the page to the active list so it is not
1192	* immediately swapped out again after swapon.
1193	*/
1194	activate_page(page);
1195	out:
1196	pte_unmap_unlock(pte, ptl);
1197	out_nolock:
1198	if (page != swapcache) {
1199	unlock_page(page);
1200	put_page(page);
1201	}
1202	return ret;
1203	}
1204
1205	static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1206	unsigned long addr, unsigned long end,
1207	swp_entry_t entry, struct page *page)
1208	{
1209	pte_t swp_pte = swp_entry_to_pte(entry);
1210	pte_t *pte;
1211	int ret = 0;
1212
1213	/*
1214	* We don't actually need pte lock while scanning for swp_pte: since
1215	* we hold page lock and mmap_sem, swp_pte cannot be inserted into the
1216	* page table while we're scanning; though it could get zapped, and on
1217	* some architectures (e.g. x86_32 with PAE) we might catch a glimpse
1218	* of unmatched parts which look like swp_pte, so unuse_pte must
1219	* recheck under pte lock. Scanning without pte lock lets it be
1220	* preemptable whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
1221	*/
1222	pte = pte_offset_map(pmd, addr);
1223	do {
1224	/*
1225	* swapoff spends a _lot_ of time in this loop!
1226	* Test inline before going to call unuse_pte.
1227	*/
1228	if (unlikely(pte_same_as_swp(*pte, swp_pte))) {
1229	pte_unmap(pte);
1230	ret = unuse_pte(vma, pmd, addr, entry, page);
1231	if (ret)
1232	goto out;
1233	pte = pte_offset_map(pmd, addr);
1234	}
1235	} while (pte++, addr += PAGE_SIZE, addr != end);
1236	pte_unmap(pte - 1);
1237	out:
1238	return ret;
1239	}
1240
1241	static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1242	unsigned long addr, unsigned long end,
1243	swp_entry_t entry, struct page *page)
1244	{
1245	pmd_t *pmd;
1246	unsigned long next;
1247	int ret;
1248
1249	pmd = pmd_offset(pud, addr);
1250	do {
1251	next = pmd_addr_end(addr, end);
1252	if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1253	continue;
1254	ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
1255	if (ret)
1256	return ret;
1257	} while (pmd++, addr = next, addr != end);
1258	return 0;
1259	}
1260
1261	static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1262	unsigned long addr, unsigned long end,
1263	swp_entry_t entry, struct page *page)
1264	{
1265	pud_t *pud;
1266	unsigned long next;
1267	int ret;
1268
1269	pud = pud_offset(pgd, addr);
1270	do {
1271	next = pud_addr_end(addr, end);
1272	if (pud_none_or_clear_bad(pud))
1273	continue;
1274	ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
1275	if (ret)
1276	return ret;
1277	} while (pud++, addr = next, addr != end);
1278	return 0;
1279	}
1280
1281	static int unuse_vma(struct vm_area_struct *vma,
1282	swp_entry_t entry, struct page *page)
1283	{
1284	pgd_t *pgd;
1285	unsigned long addr, end, next;
1286	int ret;
1287
1288	if (page_anon_vma(page)) {
1289	addr = page_address_in_vma(page, vma);
1290	if (addr == -EFAULT)
1291	return 0;
1292	else
1293	end = addr + PAGE_SIZE;
1294	} else {
1295	addr = vma->vm_start;
1296	end = vma->vm_end;
1297	}
1298
1299	pgd = pgd_offset(vma->vm_mm, addr);
1300	do {
1301	next = pgd_addr_end(addr, end);
1302	if (pgd_none_or_clear_bad(pgd))
1303	continue;
1304	ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1305	if (ret)
1306	return ret;
1307	} while (pgd++, addr = next, addr != end);
1308	return 0;
1309	}
1310
1311	static int unuse_mm(struct mm_struct *mm,
1312	swp_entry_t entry, struct page *page)
1313	{
1314	struct vm_area_struct *vma;
1315	int ret = 0;
1316
1317	if (!down_read_trylock(&mm->mmap_sem)) {
1318	/*
1319	* Activate page so shrink_inactive_list is unlikely to unmap
1320	* its ptes while lock is dropped, so swapoff can make progress.
1321	*/
1322	activate_page(page);
1323	unlock_page(page);
1324	down_read(&mm->mmap_sem);
1325	lock_page(page);
1326	}
1327	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1328	if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
1329	break;
1330	}
1331	up_read(&mm->mmap_sem);
1332	return (ret < 0)? ret: 0;
1333	}
1334
1335	/*
1336	* Scan swap_map (or frontswap_map if frontswap parameter is true)
1337	* from current position to next entry still in use.
1338	* Recycle to start on reaching the end, returning 0 when empty.
1339	*/
1340	static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1341	unsigned int prev, bool frontswap)
1342	{
1343	unsigned int max = si->max;
1344	unsigned int i = prev;
1345	unsigned char count;
1346
1347	/*
1348	* No need for swap_lock here: we're just looking
1349	* for whether an entry is in use, not modifying it; false
1350	* hits are okay, and sys_swapoff() has already prevented new
1351	* allocations from this area (while holding swap_lock).
1352	*/
1353	for (;;) {
1354	if (++i >= max) {
1355	if (!prev) {
1356	i = 0;
1357	break;
1358	}
1359	/*
1360	* No entries in use at top of swap_map,
1361	* loop back to start and recheck there.
1362	*/
1363	max = prev + 1;
1364	prev = 0;
1365	i = 1;
1366	}
1367	if (frontswap) {
1368	if (frontswap_test(si, i))
1369	break;
1370	else
1371	continue;
1372	}
1373	count = READ_ONCE(si->swap_map[i]);
1374	if (count && swap_count(count) != SWAP_MAP_BAD)
1375	break;
1376	}
1377	return i;
1378	}
1379
1380	/*
1381	* We completely avoid races by reading each swap page in advance,
1382	* and then search for the process using it. All the necessary
1383	* page table adjustments can then be made atomically.
1384	*
1385	* if the boolean frontswap is true, only unuse pages_to_unuse pages;
1386	* pages_to_unuse==0 means all pages; ignored if frontswap is false
1387	*/
1388	int try_to_unuse(unsigned int type, bool frontswap,
1389	unsigned long pages_to_unuse)
1390	{
1391	struct swap_info_struct *si = swap_info[type];
1392	struct mm_struct *start_mm;
1393	volatile unsigned char *swap_map; /* swap_map is accessed without
1394	* locking. Mark it as volatile
1395	* to prevent compiler doing
1396	* something odd.
1397	*/
1398	unsigned char swcount;
1399	struct page *page;
1400	swp_entry_t entry;
1401	unsigned int i = 0;
1402	int retval = 0;
1403
1404	/*
1405	* When searching mms for an entry, a good strategy is to
1406	* start at the first mm we freed the previous entry from
1407	* (though actually we don't notice whether we or coincidence
1408	* freed the entry). Initialize this start_mm with a hold.
1409	*
1410	* A simpler strategy would be to start at the last mm we
1411	* freed the previous entry from; but that would take less
1412	* advantage of mmlist ordering, which clusters forked mms
1413	* together, child after parent. If we race with dup_mmap(), we
1414	* prefer to resolve parent before child, lest we miss entries
1415	* duplicated after we scanned child: using last mm would invert
1416	* that.
1417	*/
1418	start_mm = &init_mm;
1419	atomic_inc(&init_mm.mm_users);
1420
1421	/*
1422	* Keep on scanning until all entries have gone. Usually,
1423	* one pass through swap_map is enough, but not necessarily:
1424	* there are races when an instance of an entry might be missed.
1425	*/
1426	while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
1427	if (signal_pending(current)) {
1428	retval = -EINTR;
1429	break;
1430	}
1431
1432	/*
1433	* Get a page for the entry, using the existing swap
1434	* cache page if there is one. Otherwise, get a clean
1435	* page and read the swap into it.
1436	*/
1437	swap_map = &si->swap_map[i];
1438	entry = swp_entry(type, i);
1439	page = read_swap_cache_async(entry,
1440	GFP_HIGHUSER_MOVABLE, NULL, 0);
1441	if (!page) {
1442	/*
1443	* Either swap_duplicate() failed because entry
1444	* has been freed independently, and will not be
1445	* reused since sys_swapoff() already disabled
1446	* allocation from here, or alloc_page() failed.
1447	*/
1448	swcount = *swap_map;
1449	/*
1450	* We don't hold lock here, so the swap entry could be
1451	* SWAP_MAP_BAD (when the cluster is discarding).
1452	* Instead of fail out, We can just skip the swap
1453	* entry because swapoff will wait for discarding
1454	* finish anyway.
1455	*/
1456	if (!swcount || swcount == SWAP_MAP_BAD)
1457	continue;
1458	retval = -ENOMEM;
1459	break;
1460	}
1461
1462	/*
1463	* Don't hold on to start_mm if it looks like exiting.
1464	*/
1465	if (atomic_read(&start_mm->mm_users) == 1) {
1466	mmput(start_mm);
1467	start_mm = &init_mm;
1468	atomic_inc(&init_mm.mm_users);
1469	}
1470
1471	/*
1472	* Wait for and lock page. When do_swap_page races with
1473	* try_to_unuse, do_swap_page can handle the fault much
1474	* faster than try_to_unuse can locate the entry. This
1475	* apparently redundant "wait_on_page_locked" lets try_to_unuse
1476	* defer to do_swap_page in such a case - in some tests,
1477	* do_swap_page and try_to_unuse repeatedly compete.
1478	*/
1479	wait_on_page_locked(page);
1480	wait_on_page_writeback(page);
1481	lock_page(page);
1482	wait_on_page_writeback(page);
1483
1484	/*
1485	* Remove all references to entry.
1486	*/
1487	swcount = *swap_map;
1488	if (swap_count(swcount) == SWAP_MAP_SHMEM) {
1489	retval = shmem_unuse(entry, page);
1490	/* page has already been unlocked and released */
1491	if (retval < 0)
1492	break;
1493	continue;
1494	}
1495	if (swap_count(swcount) && start_mm != &init_mm)
1496	retval = unuse_mm(start_mm, entry, page);
1497
1498	if (swap_count(*swap_map)) {
1499	int set_start_mm = (*swap_map >= swcount);
1500	struct list_head *p = &start_mm->mmlist;
1501	struct mm_struct *new_start_mm = start_mm;
1502	struct mm_struct *prev_mm = start_mm;
1503	struct mm_struct *mm;
1504
1505	atomic_inc(&new_start_mm->mm_users);
1506	atomic_inc(&prev_mm->mm_users);
1507	spin_lock(&mmlist_lock);
1508	while (swap_count(*swap_map) && !retval &&
1509	(p = p->next) != &start_mm->mmlist) {
1510	mm = list_entry(p, struct mm_struct, mmlist);
1511	if (!atomic_inc_not_zero(&mm->mm_users))
1512	continue;
1513	spin_unlock(&mmlist_lock);
1514	mmput(prev_mm);
1515	prev_mm = mm;
1516
1517	cond_resched();
1518
1519	swcount = *swap_map;
1520	if (!swap_count(swcount)) /* any usage ? */
1521	;
1522	else if (mm == &init_mm)
1523	set_start_mm = 1;
1524	else
1525	retval = unuse_mm(mm, entry, page);
1526
1527	if (set_start_mm && *swap_map < swcount) {
1528	mmput(new_start_mm);
1529	atomic_inc(&mm->mm_users);
1530	new_start_mm = mm;
1531	set_start_mm = 0;
1532	}
1533	spin_lock(&mmlist_lock);
1534	}
1535	spin_unlock(&mmlist_lock);
1536	mmput(prev_mm);
1537	mmput(start_mm);
1538	start_mm = new_start_mm;
1539	}
1540	if (retval) {
1541	unlock_page(page);
1542	put_page(page);
1543	break;
1544	}
1545
1546	/*
1547	* If a reference remains (rare), we would like to leave
1548	* the page in the swap cache; but try_to_unmap could
1549	* then re-duplicate the entry once we drop page lock,
1550	* so we might loop indefinitely; also, that page could
1551	* not be swapped out to other storage meanwhile. So:
1552	* delete from cache even if there's another reference,
1553	* after ensuring that the data has been saved to disk -
1554	* since if the reference remains (rarer), it will be
1555	* read from disk into another page. Splitting into two
1556	* pages would be incorrect if swap supported "shared
1557	* private" pages, but they are handled by tmpfs files.
1558	*
1559	* Given how unuse_vma() targets one particular offset
1560	* in an anon_vma, once the anon_vma has been determined,
1561	* this splitting happens to be just what is needed to
1562	* handle where KSM pages have been swapped out: re-reading
1563	* is unnecessarily slow, but we can fix that later on.
1564	*/
1565	if (swap_count(*swap_map) &&
1566	PageDirty(page) && PageSwapCache(page)) {
1567	struct writeback_control wbc = {
1568	.sync_mode = WB_SYNC_NONE,
1569	};
1570
1571	swap_writepage(page, &wbc);
1572	lock_page(page);
1573	wait_on_page_writeback(page);
1574	}
1575
1576	/*
1577	* It is conceivable that a racing task removed this page from
1578	* swap cache just before we acquired the page lock at the top,
1579	* or while we dropped it in unuse_mm(). The page might even
1580	* be back in swap cache on another swap area: that we must not
1581	* delete, since it may not have been written out to swap yet.
1582	*/
1583	if (PageSwapCache(page) &&
1584	likely(page_private(page) == entry.val))
1585	delete_from_swap_cache(page);
1586
1587	/*
1588	* So we could skip searching mms once swap count went
1589	* to 1, we did not mark any present ptes as dirty: must
1590	* mark page dirty so shrink_page_list will preserve it.
1591	*/
1592	SetPageDirty(page);
1593	unlock_page(page);
1594	put_page(page);
1595
1596	/*
1597	* Make sure that we aren't completely killing
1598	* interactive performance.
1599	*/
1600	cond_resched();
1601	if (frontswap && pages_to_unuse > 0) {
1602	if (!--pages_to_unuse)
1603	break;
1604	}
1605	}
1606
1607	mmput(start_mm);
1608	return retval;
1609	}
1610
1611	/*
1612	* After a successful try_to_unuse, if no swap is now in use, we know
1613	* we can empty the mmlist. swap_lock must be held on entry and exit.
1614	* Note that mmlist_lock nests inside swap_lock, and an mm must be
1615	* added to the mmlist just after page_duplicate - before would be racy.
1616	*/
1617	static void drain_mmlist(void)
1618	{
1619	struct list_head *p, *next;
1620	unsigned int type;
1621
1622	for (type = 0; type < nr_swapfiles; type++)
1623	if (swap_info[type]->inuse_pages)
1624	return;
1625	spin_lock(&mmlist_lock);
1626	list_for_each_safe(p, next, &init_mm.mmlist)
1627	list_del_init(p);
1628	spin_unlock(&mmlist_lock);
1629	}
1630
1631	/*
1632	* Use this swapdev's extent info to locate the (PAGE_SIZE) block which
1633	* corresponds to page offset for the specified swap entry.
1634	* Note that the type of this function is sector_t, but it returns page offset
1635	* into the bdev, not sector offset.
1636	*/
1637	static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
1638	{
1639	struct swap_info_struct *sis;
1640	struct swap_extent *start_se;
1641	struct swap_extent *se;
1642	pgoff_t offset;
1643
1644	sis = swap_info[swp_type(entry)];
1645	*bdev = sis->bdev;
1646
1647	offset = swp_offset(entry);
1648	start_se = sis->curr_swap_extent;
1649	se = start_se;
1650
1651	for ( ; ; ) {
1652	if (se->start_page <= offset &&
1653	offset < (se->start_page + se->nr_pages)) {
1654	return se->start_block + (offset - se->start_page);
1655	}
1656	se = list_next_entry(se, list);
1657	sis->curr_swap_extent = se;
1658	BUG_ON(se == start_se); /* It *must* be present */
1659	}
1660	}
1661
1662	/*
1663	* Returns the page offset into bdev for the specified page's swap entry.
1664	*/
1665	sector_t map_swap_page(struct page *page, struct block_device **bdev)
1666	{
1667	swp_entry_t entry;
1668	entry.val = page_private(page);
1669	return map_swap_entry(entry, bdev);
1670	}
1671
1672	/*
1673	* Free all of a swapdev's extent information
1674	*/
1675	static void destroy_swap_extents(struct swap_info_struct *sis)
1676	{
1677	while (!list_empty(&sis->first_swap_extent.list)) {
1678	struct swap_extent *se;
1679
1680	se = list_first_entry(&sis->first_swap_extent.list,
1681	struct swap_extent, list);
1682	list_del(&se->list);
1683	kfree(se);
1684	}
1685
1686	if (sis->flags & SWP_FILE) {
1687	struct file *swap_file = sis->swap_file;
1688	struct address_space *mapping = swap_file->f_mapping;
1689
1690	sis->flags &= ~SWP_FILE;
1691	mapping->a_ops->swap_deactivate(swap_file);
1692	}
1693	}
1694
1695	/*
1696	* Add a block range (and the corresponding page range) into this swapdev's
1697	* extent list. The extent list is kept sorted in page order.
1698	*
1699	* This function rather assumes that it is called in ascending page order.
1700	*/
1701	int
1702	add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1703	unsigned long nr_pages, sector_t start_block)
1704	{
1705	struct swap_extent *se;
1706	struct swap_extent *new_se;
1707	struct list_head *lh;
1708
1709	if (start_page == 0) {
1710	se = &sis->first_swap_extent;
1711	sis->curr_swap_extent = se;
1712	se->start_page = 0;
1713	se->nr_pages = nr_pages;
1714	se->start_block = start_block;
1715	return 1;
1716	} else {
1717	lh = sis->first_swap_extent.list.prev; /* Highest extent */
1718	se = list_entry(lh, struct swap_extent, list);
1719	BUG_ON(se->start_page + se->nr_pages != start_page);
1720	if (se->start_block + se->nr_pages == start_block) {
1721	/* Merge it */
1722	se->nr_pages += nr_pages;
1723	return 0;
1724	}
1725	}
1726
1727	/*
1728	* No merge. Insert a new extent, preserving ordering.
1729	*/
1730	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1731	if (new_se == NULL)
1732	return -ENOMEM;
1733	new_se->start_page = start_page;
1734	new_se->nr_pages = nr_pages;
1735	new_se->start_block = start_block;
1736
1737	list_add_tail(&new_se->list, &sis->first_swap_extent.list);
1738	return 1;
1739	}
1740
1741	/*
1742	* A `swap extent' is a simple thing which maps a contiguous range of pages
1743	* onto a contiguous range of disk blocks. An ordered list of swap extents
1744	* is built at swapon time and is then used at swap_writepage/swap_readpage
1745	* time for locating where on disk a page belongs.
1746	*
1747	* If the swapfile is an S_ISBLK block device, a single extent is installed.
1748	* This is done so that the main operating code can treat S_ISBLK and S_ISREG
1749	* swap files identically.
1750	*
1751	* Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
1752	* extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
1753	* swapfiles are handled *identically* after swapon time.
1754	*
1755	* For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
1756	* and will parse them into an ordered extent list, in PAGE_SIZE chunks. If
1757	* some stray blocks are found which do not fall within the PAGE_SIZE alignment
1758	* requirements, they are simply tossed out - we will never use those blocks
1759	* for swapping.
1760	*
1761	* For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This
1762	* prevents root from shooting her foot off by ftruncating an in-use swapfile,
1763	* which will scribble on the fs.
1764	*
1765	* The amount of disk space which a single swap extent represents varies.
1766	* Typically it is in the 1-4 megabyte range. So we can have hundreds of
1767	* extents in the list. To avoid much list walking, we cache the previous
1768	* search location in `curr_swap_extent', and start new searches from there.
1769	* This is extremely effective. The average number of iterations in
1770	* map_swap_page() has been measured at about 0.3 per page. - akpm.
1771	*/
1772	static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1773	{
1774	struct file *swap_file = sis->swap_file;
1775	struct address_space *mapping = swap_file->f_mapping;
1776	struct inode *inode = mapping->host;
1777	int ret;
1778
1779	if (S_ISBLK(inode->i_mode)) {
1780	ret = add_swap_extent(sis, 0, sis->max, 0);
1781	*span = sis->pages;
1782	return ret;
1783	}
1784
1785	if (mapping->a_ops->swap_activate) {
1786	ret = mapping->a_ops->swap_activate(sis, swap_file, span);
1787	if (!ret) {
1788	sis->flags |= SWP_FILE;
1789	ret = add_swap_extent(sis, 0, sis->max, 0);
1790	*span = sis->pages;
1791	}
1792	return ret;
1793	}
1794
1795	return generic_swapfile_activate(sis, swap_file, span);
1796	}
1797
1798	static void _enable_swap_info(struct swap_info_struct *p, int prio,
1799	unsigned char *swap_map,
1800	struct swap_cluster_info *cluster_info)
1801	{
1802	if (prio >= 0)
1803	p->prio = prio;
1804	else
1805	p->prio = --least_priority;
1806	/*
1807	* the plist prio is negated because plist ordering is
1808	* low-to-high, while swap ordering is high-to-low
1809	*/
1810	p->list.prio = -p->prio;
1811	p->avail_list.prio = -p->prio;
1812	p->swap_map = swap_map;
1813	p->cluster_info = cluster_info;
1814	p->flags |= SWP_WRITEOK;
1815	atomic_long_add(p->pages, &nr_swap_pages);
1816	total_swap_pages += p->pages;
1817
1818	assert_spin_locked(&swap_lock);
1819	/*
1820	* both lists are plists, and thus priority ordered.
1821	* swap_active_head needs to be priority ordered for swapoff(),
1822	* which on removal of any swap_info_struct with an auto-assigned
1823	* (i.e. negative) priority increments the auto-assigned priority
1824	* of any lower-priority swap_info_structs.
1825	* swap_avail_head needs to be priority ordered for get_swap_page(),
1826	* which allocates swap pages from the highest available priority
1827	* swap_info_struct.
1828	*/
1829	plist_add(&p->list, &swap_active_head);
1830	spin_lock(&swap_avail_lock);
1831	plist_add(&p->avail_list, &swap_avail_head);
1832	spin_unlock(&swap_avail_lock);
1833	}
1834
1835	static void enable_swap_info(struct swap_info_struct *p, int prio,
1836	unsigned char *swap_map,
1837	struct swap_cluster_info *cluster_info,
1838	unsigned long *frontswap_map)
1839	{
1840	frontswap_init(p->type, frontswap_map);
1841	spin_lock(&swap_lock);
1842	spin_lock(&p->lock);
1843	_enable_swap_info(p, prio, swap_map, cluster_info);
1844	spin_unlock(&p->lock);
1845	spin_unlock(&swap_lock);
1846	}
1847
1848	static void reinsert_swap_info(struct swap_info_struct *p)
1849	{
1850	spin_lock(&swap_lock);
1851	spin_lock(&p->lock);
1852	_enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
1853	spin_unlock(&p->lock);
1854	spin_unlock(&swap_lock);
1855	}
1856
1857	SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
1858	{
1859	struct swap_info_struct *p = NULL;
1860	unsigned char *swap_map;
1861	struct swap_cluster_info *cluster_info;
1862	unsigned long *frontswap_map;
1863	struct file *swap_file, *victim;
1864	struct address_space *mapping;
1865	struct inode *inode;
1866	struct filename *pathname;
1867	int err, found = 0;
1868	unsigned int old_block_size;
1869
1870	if (!capable(CAP_SYS_ADMIN))
1871	return -EPERM;
1872
1873	BUG_ON(!current->mm);
1874
1875	pathname = getname(specialfile);
1876	if (IS_ERR(pathname))
1877	return PTR_ERR(pathname);
1878
1879	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
1880	err = PTR_ERR(victim);
1881	if (IS_ERR(victim))
1882	goto out;
1883
1884	mapping = victim->f_mapping;
1885	spin_lock(&swap_lock);
1886	plist_for_each_entry(p, &swap_active_head, list) {
1887	if (p->flags & SWP_WRITEOK) {
1888	if (p->swap_file->f_mapping == mapping) {
1889	found = 1;
1890	break;
1891	}
1892	}
1893	}
1894	if (!found) {
1895	err = -EINVAL;
1896	spin_unlock(&swap_lock);
1897	goto out_dput;
1898	}
1899	if (!security_vm_enough_memory_mm(current->mm, p->pages))
1900	vm_unacct_memory(p->pages);
1901	else {
1902	err = -ENOMEM;
1903	spin_unlock(&swap_lock);
1904	goto out_dput;
1905	}
1906	spin_lock(&swap_avail_lock);
1907	plist_del(&p->avail_list, &swap_avail_head);
1908	spin_unlock(&swap_avail_lock);
1909	spin_lock(&p->lock);
1910	if (p->prio < 0) {
1911	struct swap_info_struct *si = p;
1912
1913	plist_for_each_entry_continue(si, &swap_active_head, list) {
1914	si->prio++;
1915	si->list.prio--;
1916	si->avail_list.prio--;
1917	}
1918	least_priority++;
1919	}
1920	plist_del(&p->list, &swap_active_head);
1921	atomic_long_sub(p->pages, &nr_swap_pages);
1922	total_swap_pages -= p->pages;
1923	p->flags &= ~SWP_WRITEOK;
1924	spin_unlock(&p->lock);
1925	spin_unlock(&swap_lock);
1926
1927	set_current_oom_origin();
1928	err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
1929	clear_current_oom_origin();
1930
1931	if (err) {
1932	/* re-insert swap space back into swap_list */
1933	reinsert_swap_info(p);
1934	goto out_dput;
1935	}
1936
1937	flush_work(&p->discard_work);
1938
1939	destroy_swap_extents(p);
1940	if (p->flags & SWP_CONTINUED)
1941	free_swap_count_continuations(p);
1942
1943	mutex_lock(&swapon_mutex);
1944	spin_lock(&swap_lock);
1945	spin_lock(&p->lock);
1946	drain_mmlist();
1947
1948	/* wait for anyone still in scan_swap_map */
1949	p->highest_bit = 0; /* cuts scans short */
1950	while (p->flags >= SWP_SCANNING) {
1951	spin_unlock(&p->lock);
1952	spin_unlock(&swap_lock);
1953	schedule_timeout_uninterruptible(1);
1954	spin_lock(&swap_lock);
1955	spin_lock(&p->lock);
1956	}
1957
1958	swap_file = p->swap_file;
1959	old_block_size = p->old_block_size;
1960	p->swap_file = NULL;
1961	p->max = 0;
1962	swap_map = p->swap_map;
1963	p->swap_map = NULL;
1964	cluster_info = p->cluster_info;
1965	p->cluster_info = NULL;
1966	frontswap_map = frontswap_map_get(p);
1967	spin_unlock(&p->lock);
1968	spin_unlock(&swap_lock);
1969	frontswap_invalidate_area(p->type);
1970	frontswap_map_set(p, NULL);
1971	mutex_unlock(&swapon_mutex);
1972	free_percpu(p->percpu_cluster);
1973	p->percpu_cluster = NULL;
1974	vfree(swap_map);
1975	vfree(cluster_info);
1976	vfree(frontswap_map);
1977	/* Destroy swap account information */
1978	swap_cgroup_swapoff(p->type);
1979
1980	inode = mapping->host;
1981	if (S_ISBLK(inode->i_mode)) {
1982	struct block_device *bdev = I_BDEV(inode);
1983	set_blocksize(bdev, old_block_size);
1984	blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
1985	} else {
1986	inode_lock(inode);
1987	inode->i_flags &= ~S_SWAPFILE;
1988	inode_unlock(inode);
1989	}
1990	filp_close(swap_file, NULL);
1991
1992	/*
1993	* Clear the SWP_USED flag after all resources are freed so that swapon
1994	* can reuse this swap_info in alloc_swap_info() safely. It is ok to
1995	* not hold p->lock after we cleared its SWP_WRITEOK.
1996	*/
1997	spin_lock(&swap_lock);
1998	p->flags = 0;
1999	spin_unlock(&swap_lock);
2000
2001	err = 0;
2002	atomic_inc(&proc_poll_event);
2003	wake_up_interruptible(&proc_poll_wait);
2004
2005	out_dput:
2006	filp_close(victim, NULL);
2007	out:
2008	putname(pathname);
2009	return err;
2010	}
2011
2012	#ifdef CONFIG_PROC_FS
2013	static unsigned swaps_poll(struct file *file, poll_table *wait)
2014	{
2015	struct seq_file *seq = file->private_data;
2016
2017	poll_wait(file, &proc_poll_wait, wait);
2018
2019	if (seq->poll_event != atomic_read(&proc_poll_event)) {
2020	seq->poll_event = atomic_read(&proc_poll_event);
2021	return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
2022	}
2023
2024	return POLLIN | POLLRDNORM;
2025	}
2026
2027	/* iterator */
2028	static void *swap_start(struct seq_file *swap, loff_t *pos)
2029	{
2030	struct swap_info_struct *si;
2031	int type;
2032	loff_t l = *pos;
2033
2034	mutex_lock(&swapon_mutex);
2035
2036	if (!l)
2037	return SEQ_START_TOKEN;
2038
2039	for (type = 0; type < nr_swapfiles; type++) {
2040	smp_rmb(); /* read nr_swapfiles before swap_info[type] */
2041	si = swap_info[type];
2042	if (!(si->flags & SWP_USED) || !si->swap_map)
2043	continue;
2044	if (!--l)
2045	return si;
2046	}
2047
2048	return NULL;
2049	}
2050
2051	static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2052	{
2053	struct swap_info_struct *si = v;
2054	int type;
2055
2056	if (v == SEQ_START_TOKEN)
2057	type = 0;
2058	else
2059	type = si->type + 1;
2060
2061	for (; type < nr_swapfiles; type++) {
2062	smp_rmb(); /* read nr_swapfiles before swap_info[type] */
2063	si = swap_info[type];
2064	if (!(si->flags & SWP_USED) || !si->swap_map)
2065	continue;
2066	++*pos;
2067	return si;
2068	}
2069
2070	return NULL;
2071	}
2072
2073	static void swap_stop(struct seq_file *swap, void *v)
2074	{
2075	mutex_unlock(&swapon_mutex);
2076	}
2077
2078	static int swap_show(struct seq_file *swap, void *v)
2079	{
2080	struct swap_info_struct *si = v;
2081	struct file *file;
2082	int len;
2083
2084	if (si == SEQ_START_TOKEN) {
2085	seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
2086	return 0;
2087	}
2088
2089	file = si->swap_file;
2090	len = seq_file_path(swap, file, " \t\n\\");
2091	seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2092	len < 40 ? 40 - len : 1, " ",
2093	S_ISBLK(file_inode(file)->i_mode) ?
2094	"partition" : "file\t",
2095	si->pages << (PAGE_SHIFT - 10),
2096	si->inuse_pages << (PAGE_SHIFT - 10),
2097	si->prio);
2098	return 0;
2099	}
2100
2101	static const struct seq_operations swaps_op = {
2102	.start = swap_start,
2103	.next = swap_next,
2104	.stop = swap_stop,
2105	.show = swap_show
2106	};
2107
2108	static int swaps_open(struct inode *inode, struct file *file)
2109	{
2110	struct seq_file *seq;
2111	int ret;
2112
2113	ret = seq_open(file, &swaps_op);
2114	if (ret)
2115	return ret;
2116
2117	seq = file->private_data;
2118	seq->poll_event = atomic_read(&proc_poll_event);
2119	return 0;
2120	}
2121
2122	static const struct file_operations proc_swaps_operations = {
2123	.open = swaps_open,
2124	.read = seq_read,
2125	.llseek = seq_lseek,
2126	.release = seq_release,
2127	.poll = swaps_poll,
2128	};
2129
2130	static int __init procswaps_init(void)
2131	{
2132	proc_create("swaps", 0, NULL, &proc_swaps_operations);
2133	return 0;
2134	}
2135	__initcall(procswaps_init);
2136	#endif /* CONFIG_PROC_FS */
2137
2138	#ifdef MAX_SWAPFILES_CHECK
2139	static int __init max_swapfiles_check(void)
2140	{
2141	MAX_SWAPFILES_CHECK();
2142	return 0;
2143	}
2144	late_initcall(max_swapfiles_check);
2145	#endif
2146
2147	static struct swap_info_struct *alloc_swap_info(void)
2148	{
2149	struct swap_info_struct *p;
2150	unsigned int type;
2151
2152	p = kzalloc(sizeof(*p), GFP_KERNEL);
2153	if (!p)
2154	return ERR_PTR(-ENOMEM);
2155
2156	spin_lock(&swap_lock);
2157	for (type = 0; type < nr_swapfiles; type++) {
2158	if (!(swap_info[type]->flags & SWP_USED))
2159	break;
2160	}
2161	if (type >= MAX_SWAPFILES) {
2162	spin_unlock(&swap_lock);
2163	kfree(p);
2164	return ERR_PTR(-EPERM);
2165	}
2166	if (type >= nr_swapfiles) {
2167	p->type = type;
2168	swap_info[type] = p;
2169	/*
2170	* Write swap_info[type] before nr_swapfiles, in case a
2171	* racing procfs swap_start() or swap_next() is reading them.
2172	* (We never shrink nr_swapfiles, we never free this entry.)
2173	*/
2174	smp_wmb();
2175	nr_swapfiles++;
2176	} else {
2177	kfree(p);
2178	p = swap_info[type];
2179	/*
2180	* Do not memset this entry: a racing procfs swap_next()
2181	* would be relying on p->type to remain valid.
2182	*/
2183	}
2184	INIT_LIST_HEAD(&p->first_swap_extent.list);
2185	plist_node_init(&p->list, 0);
2186	plist_node_init(&p->avail_list, 0);
2187	p->flags = SWP_USED;
2188	spin_unlock(&swap_lock);
2189	spin_lock_init(&p->lock);
2190
2191	return p;
2192	}
2193
2194	static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2195	{
2196	int error;
2197
2198	if (S_ISBLK(inode->i_mode)) {
2199	p->bdev = bdgrab(I_BDEV(inode));
2200	error = blkdev_get(p->bdev,
2201	FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2202	if (error < 0) {
2203	p->bdev = NULL;
2204	return error;
2205	}
2206	p->old_block_size = block_size(p->bdev);
2207	error = set_blocksize(p->bdev, PAGE_SIZE);
2208	if (error < 0)
2209	return error;
2210	p->flags |= SWP_BLKDEV;
2211	} else if (S_ISREG(inode->i_mode)) {
2212	p->bdev = inode->i_sb->s_bdev;
2213	inode_lock(inode);
2214	if (IS_SWAPFILE(inode))
2215	return -EBUSY;
2216	} else
2217	return -EINVAL;
2218
2219	return 0;
2220	}
2221
2222
2223	/*
2224	* Find out how many pages are allowed for a single swap device. There
2225	* are two limiting factors:
2226	* 1) the number of bits for the swap offset in the swp_entry_t type, and
2227	* 2) the number of bits in the swap pte, as defined by the different
2228	* architectures.
2229	*
2230	* In order to find the largest possible bit mask, a swap entry with
2231	* swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
2232	* decoded to a swp_entry_t again, and finally the swap offset is
2233	* extracted.
2234	*
2235	* This will mask all the bits from the initial ~0UL mask that can't
2236	* be encoded in either the swp_entry_t or the architecture definition
2237	* of a swap pte.
2238	*/
2239	unsigned long generic_max_swapfile_size(void)
2240	{
2241	return swp_offset(pte_to_swp_entry(
2242	swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2243	}
2244
2245	/* Can be overridden by an architecture for additional checks. */
2246	__weak unsigned long max_swapfile_size(void)
2247	{
2248	return generic_max_swapfile_size();
2249	}
2250
2251	static unsigned long read_swap_header(struct swap_info_struct *p,
2252	union swap_header *swap_header,
2253	struct inode *inode)
2254	{
2255	int i;
2256	unsigned long maxpages;
2257	unsigned long swapfilepages;
2258	unsigned long last_page;
2259
2260	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2261	pr_err("Unable to find swap-space signature\n");
2262	return 0;
2263	}
2264
2265	/* swap partition endianess hack... */
2266	if (swab32(swap_header->info.version) == 1) {
2267	swab32s(&swap_header->info.version);
2268	swab32s(&swap_header->info.last_page);
2269	swab32s(&swap_header->info.nr_badpages);
2270	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2271	return 0;
2272	for (i = 0; i < swap_header->info.nr_badpages; i++)
2273	swab32s(&swap_header->info.badpages[i]);
2274	}
2275	/* Check the swap header's sub-version */
2276	if (swap_header->info.version != 1) {
2277	pr_warn("Unable to handle swap header version %d\n",
2278	swap_header->info.version);
2279	return 0;
2280	}
2281
2282	p->lowest_bit = 1;
2283	p->cluster_next = 1;
2284	p->cluster_nr = 0;
2285
2286	maxpages = max_swapfile_size();
2287	last_page = swap_header->info.last_page;
2288	if (!last_page) {
2289	pr_warn("Empty swap-file\n");
2290	return 0;
2291	}
2292	if (last_page > maxpages) {
2293	pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2294	maxpages << (PAGE_SHIFT - 10),
2295	last_page << (PAGE_SHIFT - 10));
2296	}
2297	if (maxpages > last_page) {
2298	maxpages = last_page + 1;
2299	/* p->max is an unsigned int: don't overflow it */
2300	if ((unsigned int)maxpages == 0)
2301	maxpages = UINT_MAX;
2302	}
2303	p->highest_bit = maxpages - 1;
2304
2305	if (!maxpages)
2306	return 0;
2307	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2308	if (swapfilepages && maxpages > swapfilepages) {
2309	pr_warn("Swap area shorter than signature indicates\n");
2310	return 0;
2311	}
2312	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2313	return 0;
2314	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2315	return 0;
2316
2317	return maxpages;
2318	}
2319
2320	static int setup_swap_map_and_extents(struct swap_info_struct *p,
2321	union swap_header *swap_header,
2322	unsigned char *swap_map,
2323	struct swap_cluster_info *cluster_info,
2324	unsigned long maxpages,
2325	sector_t *span)
2326	{
2327	int i;
2328	unsigned int nr_good_pages;
2329	int nr_extents;
2330	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2331	unsigned long idx = p->cluster_next / SWAPFILE_CLUSTER;
2332
2333	nr_good_pages = maxpages - 1; /* omit header page */
2334
2335	cluster_list_init(&p->free_clusters);
2336	cluster_list_init(&p->discard_clusters);
2337
2338	for (i = 0; i < swap_header->info.nr_badpages; i++) {
2339	unsigned int page_nr = swap_header->info.badpages[i];
2340	if (page_nr == 0 || page_nr > swap_header->info.last_page)
2341	return -EINVAL;
2342	if (page_nr < maxpages) {
2343	swap_map[page_nr] = SWAP_MAP_BAD;
2344	nr_good_pages--;
2345	/*
2346	* Haven't marked the cluster free yet, no list
2347	* operation involved
2348	*/
2349	inc_cluster_info_page(p, cluster_info, page_nr);
2350	}
2351	}
2352
2353	/* Haven't marked the cluster free yet, no list operation involved */
2354	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2355	inc_cluster_info_page(p, cluster_info, i);
2356
2357	if (nr_good_pages) {
2358	swap_map[0] = SWAP_MAP_BAD;
2359	/*
2360	* Not mark the cluster free yet, no list
2361	* operation involved
2362	*/
2363	inc_cluster_info_page(p, cluster_info, 0);
2364	p->max = maxpages;
2365	p->pages = nr_good_pages;
2366	nr_extents = setup_swap_extents(p, span);
2367	if (nr_extents < 0)
2368	return nr_extents;
2369	nr_good_pages = p->pages;
2370	}
2371	if (!nr_good_pages) {
2372	pr_warn("Empty swap-file\n");
2373	return -EINVAL;
2374	}
2375
2376	if (!cluster_info)
2377	return nr_extents;
2378
2379	for (i = 0; i < nr_clusters; i++) {
2380	if (!cluster_count(&cluster_info[idx])) {
2381	cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2382	cluster_list_add_tail(&p->free_clusters, cluster_info,
2383	idx);
2384	}
2385	idx++;
2386	if (idx == nr_clusters)
2387	idx = 0;
2388	}
2389	return nr_extents;
2390	}
2391
2392	/*
2393	* Helper to sys_swapon determining if a given swap
2394	* backing device queue supports DISCARD operations.
2395	*/
2396	static bool swap_discardable(struct swap_info_struct *si)
2397	{
2398	struct request_queue *q = bdev_get_queue(si->bdev);
2399
2400	if (!q || !blk_queue_discard(q))
2401	return false;
2402
2403	return true;
2404	}
2405
2406	SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2407	{
2408	struct swap_info_struct *p;
2409	struct filename *name;
2410	struct file *swap_file = NULL;
2411	struct address_space *mapping;
2412	int prio;
2413	int error;
2414	union swap_header *swap_header;
2415	int nr_extents;
2416	sector_t span;
2417	unsigned long maxpages;
2418	unsigned char *swap_map = NULL;
2419	struct swap_cluster_info *cluster_info = NULL;
2420	unsigned long *frontswap_map = NULL;
2421	struct page *page = NULL;
2422	struct inode *inode = NULL;
2423
2424	if (swap_flags & ~SWAP_FLAGS_VALID)
2425	return -EINVAL;
2426
2427	if (!capable(CAP_SYS_ADMIN))
2428	return -EPERM;
2429
2430	p = alloc_swap_info();
2431	if (IS_ERR(p))
2432	return PTR_ERR(p);
2433
2434	INIT_WORK(&p->discard_work, swap_discard_work);
2435
2436	name = getname(specialfile);
2437	if (IS_ERR(name)) {
2438	error = PTR_ERR(name);
2439	name = NULL;
2440	goto bad_swap;
2441	}
2442	swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
2443	if (IS_ERR(swap_file)) {
2444	error = PTR_ERR(swap_file);
2445	swap_file = NULL;
2446	goto bad_swap;
2447	}
2448
2449	p->swap_file = swap_file;
2450	mapping = swap_file->f_mapping;
2451	inode = mapping->host;
2452
2453	/* If S_ISREG(inode->i_mode) will do inode_lock(inode); */
2454	error = claim_swapfile(p, inode);
2455	if (unlikely(error))
2456	goto bad_swap;
2457
2458	/*
2459	* Read the swap header.
2460	*/
2461	if (!mapping->a_ops->readpage) {
2462	error = -EINVAL;
2463	goto bad_swap;
2464	}
2465	page = read_mapping_page(mapping, 0, swap_file);
2466	if (IS_ERR(page)) {
2467	error = PTR_ERR(page);
2468	goto bad_swap;
2469	}
2470	swap_header = kmap(page);
2471
2472	maxpages = read_swap_header(p, swap_header, inode);
2473	if (unlikely(!maxpages)) {
2474	error = -EINVAL;
2475	goto bad_swap;
2476	}
2477
2478	/* OK, set up the swap map and apply the bad block list */
2479	swap_map = vzalloc(maxpages);
2480	if (!swap_map) {
2481	error = -ENOMEM;
2482	goto bad_swap;
2483	}
2484
2485	if (bdi_cap_stable_pages_required(inode_to_bdi(inode)))
2486	p->flags |= SWP_STABLE_WRITES;
2487
2488	if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
2489	int cpu;
2490
2491	p->flags |= SWP_SOLIDSTATE;
2492	/*
2493	* select a random position to start with to help wear leveling
2494	* SSD
2495	*/
2496	p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
2497
2498	cluster_info = vzalloc(DIV_ROUND_UP(maxpages,
2499	SWAPFILE_CLUSTER) * sizeof(*cluster_info));
2500	if (!cluster_info) {
2501	error = -ENOMEM;
2502	goto bad_swap;
2503	}
2504	p->percpu_cluster = alloc_percpu(struct percpu_cluster);
2505	if (!p->percpu_cluster) {
2506	error = -ENOMEM;
2507	goto bad_swap;
2508	}
2509	for_each_possible_cpu(cpu) {
2510	struct percpu_cluster *cluster;
2511	cluster = per_cpu_ptr(p->percpu_cluster, cpu);
2512	cluster_set_null(&cluster->index);
2513	}
2514	}
2515
2516	error = swap_cgroup_swapon(p->type, maxpages);
2517	if (error)
2518	goto bad_swap;
2519
2520	nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
2521	cluster_info, maxpages, &span);
2522	if (unlikely(nr_extents < 0)) {
2523	error = nr_extents;
2524	goto bad_swap;
2525	}
2526	/* frontswap enabled? set up bit-per-page map for frontswap */
2527	if (IS_ENABLED(CONFIG_FRONTSWAP))
2528	frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
2529
2530	if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
2531	/*
2532	* When discard is enabled for swap with no particular
2533	* policy flagged, we set all swap discard flags here in
2534	* order to sustain backward compatibility with older
2535	* swapon(8) releases.
2536	*/
2537	p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
2538	SWP_PAGE_DISCARD);
2539
2540	/*
2541	* By flagging sys_swapon, a sysadmin can tell us to
2542	* either do single-time area discards only, or to just
2543	* perform discards for released swap page-clusters.
2544	* Now it's time to adjust the p->flags accordingly.
2545	*/
2546	if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
2547	p->flags &= ~SWP_PAGE_DISCARD;
2548	else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
2549	p->flags &= ~SWP_AREA_DISCARD;
2550
2551	/* issue a swapon-time discard if it's still required */
2552	if (p->flags & SWP_AREA_DISCARD) {
2553	int err = discard_swap(p);
2554	if (unlikely(err))
2555	pr_err("swapon: discard_swap(%p): %d\n",
2556	p, err);
2557	}
2558	}
2559
2560	mutex_lock(&swapon_mutex);
2561	prio = -1;
2562	if (swap_flags & SWAP_FLAG_PREFER)
2563	prio =
2564	(swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2565	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2566
2567	pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2568	p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2569	nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2570	(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
2571	(p->flags & SWP_DISCARDABLE) ? "D" : "",
2572	(p->flags & SWP_AREA_DISCARD) ? "s" : "",
2573	(p->flags & SWP_PAGE_DISCARD) ? "c" : "",
2574	(frontswap_map) ? "FS" : "");
2575
2576	mutex_unlock(&swapon_mutex);
2577	atomic_inc(&proc_poll_event);
2578	wake_up_interruptible(&proc_poll_wait);
2579
2580	if (S_ISREG(inode->i_mode))
2581	inode->i_flags |= S_SWAPFILE;
2582	error = 0;
2583	goto out;
2584	bad_swap:
2585	free_percpu(p->percpu_cluster);
2586	p->percpu_cluster = NULL;
2587	if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
2588	set_blocksize(p->bdev, p->old_block_size);
2589	blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2590	}
2591	destroy_swap_extents(p);
2592	swap_cgroup_swapoff(p->type);
2593	spin_lock(&swap_lock);
2594	p->swap_file = NULL;
2595	p->flags = 0;
2596	spin_unlock(&swap_lock);
2597	vfree(swap_map);
2598	vfree(cluster_info);
2599	if (swap_file) {
2600	if (inode && S_ISREG(inode->i_mode)) {
2601	inode_unlock(inode);
2602	inode = NULL;
2603	}
2604	filp_close(swap_file, NULL);
2605	}
2606	out:
2607	if (page && !IS_ERR(page)) {
2608	kunmap(page);
2609	put_page(page);
2610	}
2611	if (name)
2612	putname(name);
2613	if (inode && S_ISREG(inode->i_mode))
2614	inode_unlock(inode);
2615	return error;
2616	}
2617
2618	void si_swapinfo(struct sysinfo *val)
2619	{
2620	unsigned int type;
2621	unsigned long nr_to_be_unused = 0;
2622
2623	spin_lock(&swap_lock);
2624	for (type = 0; type < nr_swapfiles; type++) {
2625	struct swap_info_struct *si = swap_info[type];
2626
2627	if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
2628	nr_to_be_unused += si->inuse_pages;
2629	}
2630	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
2631	val->totalswap = total_swap_pages + nr_to_be_unused;
2632	spin_unlock(&swap_lock);
2633	}
2634
2635	/*
2636	* Verify that a swap entry is valid and increment its swap map count.
2637	*
2638	* Returns error code in following case.
2639	* - success -> 0
2640	* - swp_entry is invalid -> EINVAL
2641	* - swp_entry is migration entry -> EINVAL
2642	* - swap-cache reference is requested but there is already one. -> EEXIST
2643	* - swap-cache reference is requested but the entry is not used. -> ENOENT
2644	* - swap-mapped reference requested but needs continued swap count. -> ENOMEM
2645	*/
2646	static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
2647	{
2648	struct swap_info_struct *p;
2649	unsigned long offset, type;
2650	unsigned char count;
2651	unsigned char has_cache;
2652	int err = -EINVAL;
2653
2654	if (non_swap_entry(entry))
2655	goto out;
2656
2657	type = swp_type(entry);
2658	if (type >= nr_swapfiles)
2659	goto bad_file;
2660	p = swap_info[type];
2661	offset = swp_offset(entry);
2662
2663	spin_lock(&p->lock);
2664	if (unlikely(offset >= p->max))
2665	goto unlock_out;
2666
2667	count = p->swap_map[offset];
2668
2669	/*
2670	* swapin_readahead() doesn't check if a swap entry is valid, so the
2671	* swap entry could be SWAP_MAP_BAD. Check here with lock held.
2672	*/
2673	if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
2674	err = -ENOENT;
2675	goto unlock_out;
2676	}
2677
2678	has_cache = count & SWAP_HAS_CACHE;
2679	count &= ~SWAP_HAS_CACHE;
2680	err = 0;
2681
2682	if (usage == SWAP_HAS_CACHE) {
2683
2684	/* set SWAP_HAS_CACHE if there is no cache and entry is used */
2685	if (!has_cache && count)
2686	has_cache = SWAP_HAS_CACHE;
2687	else if (has_cache) /* someone else added cache */
2688	err = -EEXIST;
2689	else /* no users remaining */
2690	err = -ENOENT;
2691
2692	} else if (count || has_cache) {
2693
2694	if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
2695	count += usage;
2696	else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
2697	err = -EINVAL;
2698	else if (swap_count_continued(p, offset, count))
2699	count = COUNT_CONTINUED;
2700	else
2701	err = -ENOMEM;
2702	} else
2703	err = -ENOENT; /* unused swap entry */
2704
2705	p->swap_map[offset] = count | has_cache;
2706
2707	unlock_out:
2708	spin_unlock(&p->lock);
2709	out:
2710	return err;
2711
2712	bad_file:
2713	pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
2714	goto out;
2715	}
2716
2717	/*
2718	* Help swapoff by noting that swap entry belongs to shmem/tmpfs
2719	* (in which case its reference count is never incremented).
2720	*/
2721	void swap_shmem_alloc(swp_entry_t entry)
2722	{
2723	__swap_duplicate(entry, SWAP_MAP_SHMEM);
2724	}
2725
2726	/*
2727	* Increase reference count of swap entry by 1.
2728	* Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
2729	* but could not be atomically allocated. Returns 0, just as if it succeeded,
2730	* if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
2731	* might occur if a page table entry has got corrupted.
2732	*/
2733	int swap_duplicate(swp_entry_t entry)
2734	{
2735	int err = 0;
2736
2737	while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
2738	err = add_swap_count_continuation(entry, GFP_ATOMIC);
2739	return err;
2740	}
2741
2742	/*
2743	* @entry: swap entry for which we allocate swap cache.
2744	*
2745	* Called when allocating swap cache for existing swap entry,
2746	* This can return error codes. Returns 0 at success.
2747	* -EBUSY means there is a swap cache.
2748	* Note: return code is different from swap_duplicate().
2749	*/
2750	int swapcache_prepare(swp_entry_t entry)
2751	{
2752	return __swap_duplicate(entry, SWAP_HAS_CACHE);
2753	}
2754
2755	struct swap_info_struct *page_swap_info(struct page *page)
2756	{
2757	swp_entry_t swap = { .val = page_private(page) };
2758	return swap_info[swp_type(swap)];
2759	}
2760
2761	/*
2762	* out-of-line __page_file_ methods to avoid include hell.
2763	*/
2764	struct address_space *__page_file_mapping(struct page *page)
2765	{
2766	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2767	return page_swap_info(page)->swap_file->f_mapping;
2768	}
2769	EXPORT_SYMBOL_GPL(__page_file_mapping);
2770
2771	pgoff_t __page_file_index(struct page *page)
2772	{
2773	swp_entry_t swap = { .val = page_private(page) };
2774	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2775	return swp_offset(swap);
2776	}
2777	EXPORT_SYMBOL_GPL(__page_file_index);
2778
2779	/*
2780	* add_swap_count_continuation - called when a swap count is duplicated
2781	* beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
2782	* page of the original vmalloc'ed swap_map, to hold the continuation count
2783	* (for that entry and for its neighbouring PAGE_SIZE swap entries). Called
2784	* again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
2785	*
2786	* These continuation pages are seldom referenced: the common paths all work
2787	* on the original swap_map, only referring to a continuation page when the
2788	* low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
2789	*
2790	* add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
2791	* page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
2792	* can be called after dropping locks.
2793	*/
2794	int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
2795	{
2796	struct swap_info_struct *si;
2797	struct page *head;
2798	struct page *page;
2799	struct page *list_page;
2800	pgoff_t offset;
2801	unsigned char count;
2802
2803	/*
2804	* When debugging, it's easier to use __GFP_ZERO here; but it's better
2805	* for latency not to zero a page while GFP_ATOMIC and holding locks.
2806	*/
2807	page = alloc_page(gfp_mask | __GFP_HIGHMEM);
2808
2809	si = swap_info_get(entry);
2810	if (!si) {
2811	/*
2812	* An acceptable race has occurred since the failing
2813	* __swap_duplicate(): the swap entry has been freed,
2814	* perhaps even the whole swap_map cleared for swapoff.
2815	*/
2816	goto outer;
2817	}
2818
2819	offset = swp_offset(entry);
2820	count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
2821
2822	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
2823	/*
2824	* The higher the swap count, the more likely it is that tasks
2825	* will race to add swap count continuation: we need to avoid
2826	* over-provisioning.
2827	*/
2828	goto out;
2829	}
2830
2831	if (!page) {
2832	spin_unlock(&si->lock);
2833	return -ENOMEM;
2834	}
2835
2836	/*
2837	* We are fortunate that although vmalloc_to_page uses pte_offset_map,
2838	* no architecture is using highmem pages for kernel page tables: so it
2839	* will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
2840	*/
2841	head = vmalloc_to_page(si->swap_map + offset);
2842	offset &= ~PAGE_MASK;
2843
2844	/*
2845	* Page allocation does not initialize the page's lru field,
2846	* but it does always reset its private field.
2847	*/
2848	if (!page_private(head)) {
2849	BUG_ON(count & COUNT_CONTINUED);
2850	INIT_LIST_HEAD(&head->lru);
2851	set_page_private(head, SWP_CONTINUED);
2852	si->flags |= SWP_CONTINUED;
2853	}
2854
2855	list_for_each_entry(list_page, &head->lru, lru) {
2856	unsigned char *map;
2857
2858	/*
2859	* If the previous map said no continuation, but we've found
2860	* a continuation page, free our allocation and use this one.
2861	*/
2862	if (!(count & COUNT_CONTINUED))
2863	goto out;
2864
2865	map = kmap_atomic(list_page) + offset;
2866	count = *map;
2867	kunmap_atomic(map);
2868
2869	/*
2870	* If this continuation count now has some space in it,
2871	* free our allocation and use this one.
2872	*/
2873	if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
2874	goto out;
2875	}
2876
2877	list_add_tail(&page->lru, &head->lru);
2878	page = NULL; /* now it's attached, don't free it */
2879	out:
2880	spin_unlock(&si->lock);
2881	outer:
2882	if (page)
2883	__free_page(page);
2884	return 0;
2885	}
2886
2887	/*
2888	* swap_count_continued - when the original swap_map count is incremented
2889	* from SWAP_MAP_MAX, check if there is already a continuation page to carry
2890	* into, carry if so, or else fail until a new continuation page is allocated;
2891	* when the original swap_map count is decremented from 0 with continuation,
2892	* borrow from the continuation and report whether it still holds more.
2893	* Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
2894	*/
2895	static bool swap_count_continued(struct swap_info_struct *si,
2896	pgoff_t offset, unsigned char count)
2897	{
2898	struct page *head;
2899	struct page *page;
2900	unsigned char *map;
2901
2902	head = vmalloc_to_page(si->swap_map + offset);
2903	if (page_private(head) != SWP_CONTINUED) {
2904	BUG_ON(count & COUNT_CONTINUED);
2905	return false; /* need to add count continuation */
2906	}
2907
2908	offset &= ~PAGE_MASK;
2909	page = list_entry(head->lru.next, struct page, lru);
2910	map = kmap_atomic(page) + offset;
2911
2912	if (count == SWAP_MAP_MAX) /* initial increment from swap_map */
2913	goto init_map; /* jump over SWAP_CONT_MAX checks */
2914
2915	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
2916	/*
2917	* Think of how you add 1 to 999
2918	*/
2919	while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
2920	kunmap_atomic(map);
2921	page = list_entry(page->lru.next, struct page, lru);
2922	BUG_ON(page == head);
2923	map = kmap_atomic(page) + offset;
2924	}
2925	if (*map == SWAP_CONT_MAX) {
2926	kunmap_atomic(map);
2927	page = list_entry(page->lru.next, struct page, lru);
2928	if (page == head)
2929	return false; /* add count continuation */
2930	map = kmap_atomic(page) + offset;
2931	init_map: *map = 0; /* we didn't zero the page */
2932	}
2933	*map += 1;
2934	kunmap_atomic(map);
2935	page = list_entry(page->lru.prev, struct page, lru);
2936	while (page != head) {
2937	map = kmap_atomic(page) + offset;
2938	*map = COUNT_CONTINUED;
2939	kunmap_atomic(map);
2940	page = list_entry(page->lru.prev, struct page, lru);
2941	}
2942	return true; /* incremented */
2943
2944	} else { /* decrementing */
2945	/*
2946	* Think of how you subtract 1 from 1000
2947	*/
2948	BUG_ON(count != COUNT_CONTINUED);
2949	while (*map == COUNT_CONTINUED) {
2950	kunmap_atomic(map);
2951	page = list_entry(page->lru.next, struct page, lru);
2952	BUG_ON(page == head);
2953	map = kmap_atomic(page) + offset;
2954	}
2955	BUG_ON(*map == 0);
2956	*map -= 1;
2957	if (*map == 0)
2958	count = 0;
2959	kunmap_atomic(map);
2960	page = list_entry(page->lru.prev, struct page, lru);
2961	while (page != head) {
2962	map = kmap_atomic(page) + offset;
2963	*map = SWAP_CONT_MAX | count;
2964	count = COUNT_CONTINUED;
2965	kunmap_atomic(map);
2966	page = list_entry(page->lru.prev, struct page, lru);
2967	}
2968	return count == COUNT_CONTINUED;
2969	}
2970	}
2971
2972	/*
2973	* free_swap_count_continuations - swapoff free all the continuation pages
2974	* appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
2975	*/
2976	static void free_swap_count_continuations(struct swap_info_struct *si)
2977	{
2978	pgoff_t offset;
2979
2980	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
2981	struct page *head;
2982	head = vmalloc_to_page(si->swap_map + offset);
2983	if (page_private(head)) {
2984	struct page *page, *next;
2985
2986	list_for_each_entry_safe(page, next, &head->lru, lru) {
2987	list_del(&page->lru);
2988	__free_page(page);
2989	}
2990	}
2991	}
2992	}
2993