path: root/mm/filemap.c (plain)
blob: 01a0d6a5358eefa72f24b30dd6bcfa7fbedea6ed

1	/*
2	* linux/mm/filemap.c
3	*
4	* Copyright (C) 1994-1999 Linus Torvalds
5	*/
6
7	/*
8	* This file handles the generic file mmap semantics used by
9	* most "normal" filesystems (but you don't /have/ to use this:
10	* the NFS filesystem used to do this differently, for example)
11	*/
12	#include <linux/export.h>
13	#include <linux/compiler.h>
14	#include <linux/dax.h>
15	#include <linux/fs.h>
16	#include <linux/uaccess.h>
17	#include <linux/capability.h>
18	#include <linux/kernel_stat.h>
19	#include <linux/gfp.h>
20	#include <linux/mm.h>
21	#include <linux/swap.h>
22	#include <linux/mman.h>
23	#include <linux/pagemap.h>
24	#include <linux/file.h>
25	#include <linux/uio.h>
26	#include <linux/hash.h>
27	#include <linux/writeback.h>
28	#include <linux/backing-dev.h>
29	#include <linux/pagevec.h>
30	#include <linux/blkdev.h>
31	#include <linux/security.h>
32	#include <linux/cpuset.h>
33	#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
34	#include <linux/hugetlb.h>
35	#include <linux/memcontrol.h>
36	#include <linux/cleancache.h>
37	#include <linux/rmap.h>
38	#include <linux/delayacct.h>
39	#include <linux/psi.h>
40	#include "internal.h"
41
42	#define CREATE_TRACE_POINTS
43	#include <trace/events/filemap.h>
44
45	/*
46	* FIXME: remove all knowledge of the buffer layer from the core VM
47	*/
48	#include <linux/buffer_head.h> /* for try_to_free_buffers */
49
50	#include <asm/mman.h>
51
52	/*
53	* Shared mappings implemented 30.11.1994. It's not fully working yet,
54	* though.
55	*
56	* Shared mappings now work. 15.8.1995 Bruno.
57	*
58	* finished 'unifying' the page and buffer cache and SMP-threaded the
59	* page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
60	*
61	* SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
62	*/
63
64	/*
65	* Lock ordering:
66	*
67	* ->i_mmap_rwsem (truncate_pagecache)
68	* ->private_lock (__free_pte->__set_page_dirty_buffers)
69	* ->swap_lock (exclusive_swap_page, others)
70	* ->mapping->tree_lock
71	*
72	* ->i_mutex
73	* ->i_mmap_rwsem (truncate->unmap_mapping_range)
74	*
75	* ->mmap_sem
76	* ->i_mmap_rwsem
77	* ->page_table_lock or pte_lock (various, mainly in memory.c)
78	* ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
79	*
80	* ->mmap_sem
81	* ->lock_page (access_process_vm)
82	*
83	* ->i_mutex (generic_perform_write)
84	* ->mmap_sem (fault_in_pages_readable->do_page_fault)
85	*
86	* bdi->wb.list_lock
87	* sb_lock (fs/fs-writeback.c)
88	* ->mapping->tree_lock (__sync_single_inode)
89	*
90	* ->i_mmap_rwsem
91	* ->anon_vma.lock (vma_adjust)
92	*
93	* ->anon_vma.lock
94	* ->page_table_lock or pte_lock (anon_vma_prepare and various)
95	*
96	* ->page_table_lock or pte_lock
97	* ->swap_lock (try_to_unmap_one)
98	* ->private_lock (try_to_unmap_one)
99	* ->tree_lock (try_to_unmap_one)
100	* ->zone_lru_lock(zone) (follow_page->mark_page_accessed)
101	* ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page)
102	* ->private_lock (page_remove_rmap->set_page_dirty)
103	* ->tree_lock (page_remove_rmap->set_page_dirty)
104	* bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
105	* ->inode->i_lock (page_remove_rmap->set_page_dirty)
106	* ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
107	* bdi.wb->list_lock (zap_pte_range->set_page_dirty)
108	* ->inode->i_lock (zap_pte_range->set_page_dirty)
109	* ->private_lock (zap_pte_range->__set_page_dirty_buffers)
110	*
111	* ->i_mmap_rwsem
112	* ->tasklist_lock (memory_failure, collect_procs_ao)
113	*/
114
115	static int page_cache_tree_insert(struct address_space *mapping,
116	struct page *page, void **shadowp)
117	{
118	struct radix_tree_node *node;
119	void **slot;
120	int error;
121
122	error = __radix_tree_create(&mapping->page_tree, page->index, 0,
123	&node, &slot);
124	if (error)
125	return error;
126	if (*slot) {
127	void *p;
128
129	p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
130	if (!radix_tree_exceptional_entry(p))
131	return -EEXIST;
132
133	mapping->nrexceptional--;
134	if (!dax_mapping(mapping)) {
135	if (shadowp)
136	*shadowp = p;
137	if (node)
138	workingset_node_shadows_dec(node);
139	} else {
140	/* DAX can replace empty locked entry with a hole */
141	WARN_ON_ONCE(p !=
142	(void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
143	RADIX_DAX_ENTRY_LOCK));
144	/* DAX accounts exceptional entries as normal pages */
145	if (node)
146	workingset_node_pages_dec(node);
147	/* Wakeup waiters for exceptional entry lock */
148	dax_wake_mapping_entry_waiter(mapping, page->index,
149	true);
150	}
151	}
152	radix_tree_replace_slot(slot, page);
153	mapping->nrpages++;
154	if (node) {
155	workingset_node_pages_inc(node);
156	/*
157	* Don't track node that contains actual pages.
158	*
159	* Avoid acquiring the list_lru lock if already
160	* untracked. The list_empty() test is safe as
161	* node->private_list is protected by
162	* mapping->tree_lock.
163	*/
164	if (!list_empty(&node->private_list))
165	list_lru_del(&workingset_shadow_nodes,
166	&node->private_list);
167	}
168	return 0;
169	}
170
171	static void page_cache_tree_delete(struct address_space *mapping,
172	struct page *page, void *shadow)
173	{
174	int i, nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
175
176	VM_BUG_ON_PAGE(!PageLocked(page), page);
177	VM_BUG_ON_PAGE(PageTail(page), page);
178	VM_BUG_ON_PAGE(nr != 1 && shadow, page);
179
180	for (i = 0; i < nr; i++) {
181	struct radix_tree_node *node;
182	void **slot;
183
184	__radix_tree_lookup(&mapping->page_tree, page->index + i,
185	&node, &slot);
186
187	radix_tree_clear_tags(&mapping->page_tree, node, slot);
188
189	if (!node) {
190	VM_BUG_ON_PAGE(nr != 1, page);
191	/*
192	* We need a node to properly account shadow
193	* entries. Don't plant any without. XXX
194	*/
195	shadow = NULL;
196	}
197
198	radix_tree_replace_slot(slot, shadow);
199
200	if (!node)
201	break;
202
203	workingset_node_pages_dec(node);
204	if (shadow)
205	workingset_node_shadows_inc(node);
206	else
207	if (__radix_tree_delete_node(&mapping->page_tree, node))
208	continue;
209
210	/*
211	* Track node that only contains shadow entries. DAX mappings
212	* contain no shadow entries and may contain other exceptional
213	* entries so skip those.
214	*
215	* Avoid acquiring the list_lru lock if already tracked.
216	* The list_empty() test is safe as node->private_list is
217	* protected by mapping->tree_lock.
218	*/
219	if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
220	list_empty(&node->private_list)) {
221	node->private_data = mapping;
222	list_lru_add(&workingset_shadow_nodes,
223	&node->private_list);
224	}
225	}
226
227	if (shadow) {
228	mapping->nrexceptional += nr;
229	/*
230	* Make sure the nrexceptional update is committed before
231	* the nrpages update so that final truncate racing
232	* with reclaim does not see both counters 0 at the
233	* same time and miss a shadow entry.
234	*/
235	smp_wmb();
236	}
237	mapping->nrpages -= nr;
238	}
239
240	/*
241	* Delete a page from the page cache and free it. Caller has to make
242	* sure the page is locked and that nobody else uses it - or that usage
243	* is safe. The caller must hold the mapping's tree_lock.
244	*/
245	void __delete_from_page_cache(struct page *page, void *shadow)
246	{
247	struct address_space *mapping = page->mapping;
248	int nr = hpage_nr_pages(page);
249
250	trace_mm_filemap_delete_from_page_cache(page);
251	/*
252	* if we're uptodate, flush out into the cleancache, otherwise
253	* invalidate any existing cleancache entries. We can't leave
254	* stale data around in the cleancache once our page is gone
255	*/
256	if (PageUptodate(page) && PageMappedToDisk(page))
257	cleancache_put_page(page);
258	else
259	cleancache_invalidate_page(mapping, page);
260
261	VM_BUG_ON_PAGE(PageTail(page), page);
262	VM_BUG_ON_PAGE(page_mapped(page), page);
263	if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
264	int mapcount;
265
266	pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
267	current->comm, page_to_pfn(page));
268	dump_page(page, "still mapped when deleted");
269	dump_stack();
270	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
271
272	mapcount = page_mapcount(page);
273	if (mapping_exiting(mapping) &&
274	page_count(page) >= mapcount + 2) {
275	/*
276	* All vmas have already been torn down, so it's
277	* a good bet that actually the page is unmapped,
278	* and we'd prefer not to leak it: if we're wrong,
279	* some other bad page check should catch it later.
280	*/
281	page_mapcount_reset(page);
282	page_ref_sub(page, mapcount);
283	}
284	}
285
286	page_cache_tree_delete(mapping, page, shadow);
287
288	page->mapping = NULL;
289	/* Leave page->index set: truncation lookup relies upon it */
290
291	/* hugetlb pages do not participate in page cache accounting. */
292	if (!PageHuge(page))
293	__mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
294	if (PageSwapBacked(page)) {
295	__mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
296	if (PageTransHuge(page))
297	__dec_node_page_state(page, NR_SHMEM_THPS);
298	} else {
299	VM_BUG_ON_PAGE(PageTransHuge(page) && !PageHuge(page), page);
300	}
301
302	/*
303	* At this point page must be either written or cleaned by truncate.
304	* Dirty page here signals a bug and loss of unwritten data.
305	*
306	* This fixes dirty accounting after removing the page entirely but
307	* leaves PageDirty set: it has no effect for truncated page and
308	* anyway will be cleared before returning page into buddy allocator.
309	*/
310	if (WARN_ON_ONCE(PageDirty(page)))
311	account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
312	}
313
314	/**
315	* delete_from_page_cache - delete page from page cache
316	* @page: the page which the kernel is trying to remove from page cache
317	*
318	* This must be called only on pages that have been verified to be in the page
319	* cache and locked. It will never put the page into the free list, the caller
320	* has a reference on the page.
321	*/
322	void delete_from_page_cache(struct page *page)
323	{
324	struct address_space *mapping = page_mapping(page);
325	unsigned long flags;
326	void (*freepage)(struct page *);
327
328	BUG_ON(!PageLocked(page));
329
330	freepage = mapping->a_ops->freepage;
331
332	spin_lock_irqsave(&mapping->tree_lock, flags);
333	__delete_from_page_cache(page, NULL);
334	spin_unlock_irqrestore(&mapping->tree_lock, flags);
335
336	if (freepage)
337	freepage(page);
338
339	if (PageTransHuge(page) && !PageHuge(page)) {
340	page_ref_sub(page, HPAGE_PMD_NR);
341	VM_BUG_ON_PAGE(page_count(page) <= 0, page);
342	} else {
343	put_page(page);
344	}
345	}
346	EXPORT_SYMBOL(delete_from_page_cache);
347
348	int filemap_check_errors(struct address_space *mapping)
349	{
350	int ret = 0;
351	/* Check for outstanding write errors */
352	if (test_bit(AS_ENOSPC, &mapping->flags) &&
353	test_and_clear_bit(AS_ENOSPC, &mapping->flags))
354	ret = -ENOSPC;
355	if (test_bit(AS_EIO, &mapping->flags) &&
356	test_and_clear_bit(AS_EIO, &mapping->flags))
357	ret = -EIO;
358	return ret;
359	}
360	EXPORT_SYMBOL(filemap_check_errors);
361
362	/**
363	* __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
364	* @mapping: address space structure to write
365	* @start: offset in bytes where the range starts
366	* @end: offset in bytes where the range ends (inclusive)
367	* @sync_mode: enable synchronous operation
368	*
369	* Start writeback against all of a mapping's dirty pages that lie
370	* within the byte offsets <start, end> inclusive.
371	*
372	* If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
373	* opposed to a regular memory cleansing writeback. The difference between
374	* these two operations is that if a dirty page/buffer is encountered, it must
375	* be waited upon, and not just skipped over.
376	*/
377	int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
378	loff_t end, int sync_mode)
379	{
380	int ret;
381	struct writeback_control wbc = {
382	.sync_mode = sync_mode,
383	.nr_to_write = LONG_MAX,
384	.range_start = start,
385	.range_end = end,
386	};
387
388	if (!mapping_cap_writeback_dirty(mapping))
389	return 0;
390
391	wbc_attach_fdatawrite_inode(&wbc, mapping->host);
392	ret = do_writepages(mapping, &wbc);
393	wbc_detach_inode(&wbc);
394	return ret;
395	}
396
397	static inline int __filemap_fdatawrite(struct address_space *mapping,
398	int sync_mode)
399	{
400	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
401	}
402
403	int filemap_fdatawrite(struct address_space *mapping)
404	{
405	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
406	}
407	EXPORT_SYMBOL(filemap_fdatawrite);
408
409	int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
410	loff_t end)
411	{
412	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
413	}
414	EXPORT_SYMBOL(filemap_fdatawrite_range);
415
416	/**
417	* filemap_flush - mostly a non-blocking flush
418	* @mapping: target address_space
419	*
420	* This is a mostly non-blocking flush. Not suitable for data-integrity
421	* purposes - I/O may not be started against all dirty pages.
422	*/
423	int filemap_flush(struct address_space *mapping)
424	{
425	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
426	}
427	EXPORT_SYMBOL(filemap_flush);
428
429	static int __filemap_fdatawait_range(struct address_space *mapping,
430	loff_t start_byte, loff_t end_byte)
431	{
432	pgoff_t index = start_byte >> PAGE_SHIFT;
433	pgoff_t end = end_byte >> PAGE_SHIFT;
434	struct pagevec pvec;
435	int nr_pages;
436	int ret = 0;
437
438	if (end_byte < start_byte)
439	goto out;
440
441	pagevec_init(&pvec, 0);
442	while (index <= end) {
443	unsigned i;
444
445	nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
446	end, PAGECACHE_TAG_WRITEBACK);
447	if (!nr_pages)
448	break;
449
450	for (i = 0; i < nr_pages; i++) {
451	struct page *page = pvec.pages[i];
452
453	wait_on_page_writeback(page);
454	if (TestClearPageError(page))
455	ret = -EIO;
456	}
457	pagevec_release(&pvec);
458	cond_resched();
459	}
460	out:
461	return ret;
462	}
463
464	/**
465	* filemap_fdatawait_range - wait for writeback to complete
466	* @mapping: address space structure to wait for
467	* @start_byte: offset in bytes where the range starts
468	* @end_byte: offset in bytes where the range ends (inclusive)
469	*
470	* Walk the list of under-writeback pages of the given address space
471	* in the given range and wait for all of them. Check error status of
472	* the address space and return it.
473	*
474	* Since the error status of the address space is cleared by this function,
475	* callers are responsible for checking the return value and handling and/or
476	* reporting the error.
477	*/
478	int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
479	loff_t end_byte)
480	{
481	int ret, ret2;
482
483	ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
484	ret2 = filemap_check_errors(mapping);
485	if (!ret)
486	ret = ret2;
487
488	return ret;
489	}
490	EXPORT_SYMBOL(filemap_fdatawait_range);
491
492	/**
493	* filemap_fdatawait_keep_errors - wait for writeback without clearing errors
494	* @mapping: address space structure to wait for
495	*
496	* Walk the list of under-writeback pages of the given address space
497	* and wait for all of them. Unlike filemap_fdatawait(), this function
498	* does not clear error status of the address space.
499	*
500	* Use this function if callers don't handle errors themselves. Expected
501	* call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
502	* fsfreeze(8)
503	*/
504	void filemap_fdatawait_keep_errors(struct address_space *mapping)
505	{
506	loff_t i_size = i_size_read(mapping->host);
507
508	if (i_size == 0)
509	return;
510
511	__filemap_fdatawait_range(mapping, 0, i_size - 1);
512	}
513
514	/**
515	* filemap_fdatawait - wait for all under-writeback pages to complete
516	* @mapping: address space structure to wait for
517	*
518	* Walk the list of under-writeback pages of the given address space
519	* and wait for all of them. Check error status of the address space
520	* and return it.
521	*
522	* Since the error status of the address space is cleared by this function,
523	* callers are responsible for checking the return value and handling and/or
524	* reporting the error.
525	*/
526	int filemap_fdatawait(struct address_space *mapping)
527	{
528	loff_t i_size = i_size_read(mapping->host);
529
530	if (i_size == 0)
531	return 0;
532
533	return filemap_fdatawait_range(mapping, 0, i_size - 1);
534	}
535	EXPORT_SYMBOL(filemap_fdatawait);
536
537	int filemap_write_and_wait(struct address_space *mapping)
538	{
539	int err = 0;
540
541	if ((!dax_mapping(mapping) && mapping->nrpages) ||
542	(dax_mapping(mapping) && mapping->nrexceptional)) {
543	err = filemap_fdatawrite(mapping);
544	/*
545	* Even if the above returned error, the pages may be
546	* written partially (e.g. -ENOSPC), so we wait for it.
547	* But the -EIO is special case, it may indicate the worst
548	* thing (e.g. bug) happened, so we avoid waiting for it.
549	*/
550	if (err != -EIO) {
551	int err2 = filemap_fdatawait(mapping);
552	if (!err)
553	err = err2;
554	}
555	} else {
556	err = filemap_check_errors(mapping);
557	}
558	return err;
559	}
560	EXPORT_SYMBOL(filemap_write_and_wait);
561
562	/**
563	* filemap_write_and_wait_range - write out & wait on a file range
564	* @mapping: the address_space for the pages
565	* @lstart: offset in bytes where the range starts
566	* @lend: offset in bytes where the range ends (inclusive)
567	*
568	* Write out and wait upon file offsets lstart->lend, inclusive.
569	*
570	* Note that `lend' is inclusive (describes the last byte to be written) so
571	* that this function can be used to write to the very end-of-file (end = -1).
572	*/
573	int filemap_write_and_wait_range(struct address_space *mapping,
574	loff_t lstart, loff_t lend)
575	{
576	int err = 0;
577
578	if ((!dax_mapping(mapping) && mapping->nrpages) ||
579	(dax_mapping(mapping) && mapping->nrexceptional)) {
580	err = __filemap_fdatawrite_range(mapping, lstart, lend,
581	WB_SYNC_ALL);
582	/* See comment of filemap_write_and_wait() */
583	if (err != -EIO) {
584	int err2 = filemap_fdatawait_range(mapping,
585	lstart, lend);
586	if (!err)
587	err = err2;
588	}
589	} else {
590	err = filemap_check_errors(mapping);
591	}
592	return err;
593	}
594	EXPORT_SYMBOL(filemap_write_and_wait_range);
595
596	/**
597	* replace_page_cache_page - replace a pagecache page with a new one
598	* @old: page to be replaced
599	* @new: page to replace with
600	* @gfp_mask: allocation mode
601	*
602	* This function replaces a page in the pagecache with a new one. On
603	* success it acquires the pagecache reference for the new page and
604	* drops it for the old page. Both the old and new pages must be
605	* locked. This function does not add the new page to the LRU, the
606	* caller must do that.
607	*
608	* The remove + add is atomic. The only way this function can fail is
609	* memory allocation failure.
610	*/
611	int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
612	{
613	int error;
614
615	VM_BUG_ON_PAGE(!PageLocked(old), old);
616	VM_BUG_ON_PAGE(!PageLocked(new), new);
617	VM_BUG_ON_PAGE(new->mapping, new);
618
619	error = radix_tree_preload(gfp_mask & GFP_RECLAIM_MASK);
620	if (!error) {
621	struct address_space *mapping = old->mapping;
622	void (*freepage)(struct page *);
623	unsigned long flags;
624
625	pgoff_t offset = old->index;
626	freepage = mapping->a_ops->freepage;
627
628	get_page(new);
629	new->mapping = mapping;
630	new->index = offset;
631
632	spin_lock_irqsave(&mapping->tree_lock, flags);
633	__delete_from_page_cache(old, NULL);
634	error = page_cache_tree_insert(mapping, new, NULL);
635	BUG_ON(error);
636
637	/*
638	* hugetlb pages do not participate in page cache accounting.
639	*/
640	if (!PageHuge(new))
641	__inc_node_page_state(new, NR_FILE_PAGES);
642	if (PageSwapBacked(new))
643	__inc_node_page_state(new, NR_SHMEM);
644	spin_unlock_irqrestore(&mapping->tree_lock, flags);
645	mem_cgroup_migrate(old, new);
646	radix_tree_preload_end();
647	if (freepage)
648	freepage(old);
649	put_page(old);
650	}
651
652	return error;
653	}
654	EXPORT_SYMBOL_GPL(replace_page_cache_page);
655
656	static int __add_to_page_cache_locked(struct page *page,
657	struct address_space *mapping,
658	pgoff_t offset, gfp_t gfp_mask,
659	void **shadowp)
660	{
661	int huge = PageHuge(page);
662	struct mem_cgroup *memcg;
663	int error;
664
665	VM_BUG_ON_PAGE(!PageLocked(page), page);
666	VM_BUG_ON_PAGE(PageSwapBacked(page), page);
667
668	if (!huge) {
669	error = mem_cgroup_try_charge(page, current->mm,
670	gfp_mask, &memcg, false);
671	if (error)
672	return error;
673	}
674
675	error = radix_tree_maybe_preload(gfp_mask & GFP_RECLAIM_MASK);
676	if (error) {
677	if (!huge)
678	mem_cgroup_cancel_charge(page, memcg, false);
679	return error;
680	}
681
682	get_page(page);
683	page->mapping = mapping;
684	page->index = offset;
685
686	spin_lock_irq(&mapping->tree_lock);
687	error = page_cache_tree_insert(mapping, page, shadowp);
688	radix_tree_preload_end();
689	if (unlikely(error))
690	goto err_insert;
691
692	/* hugetlb pages do not participate in page cache accounting. */
693	if (!huge)
694	__inc_node_page_state(page, NR_FILE_PAGES);
695	spin_unlock_irq(&mapping->tree_lock);
696	if (!huge)
697	mem_cgroup_commit_charge(page, memcg, false, false);
698	trace_mm_filemap_add_to_page_cache(page);
699	return 0;
700	err_insert:
701	page->mapping = NULL;
702	/* Leave page->index set: truncation relies upon it */
703	spin_unlock_irq(&mapping->tree_lock);
704	if (!huge)
705	mem_cgroup_cancel_charge(page, memcg, false);
706	put_page(page);
707	return error;
708	}
709
710	/**
711	* add_to_page_cache_locked - add a locked page to the pagecache
712	* @page: page to add
713	* @mapping: the page's address_space
714	* @offset: page index
715	* @gfp_mask: page allocation mode
716	*
717	* This function is used to add a page to the pagecache. It must be locked.
718	* This function does not add the page to the LRU. The caller must do that.
719	*/
720	int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
721	pgoff_t offset, gfp_t gfp_mask)
722	{
723	return __add_to_page_cache_locked(page, mapping, offset,
724	gfp_mask, NULL);
725	}
726	EXPORT_SYMBOL(add_to_page_cache_locked);
727
728	int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
729	pgoff_t offset, gfp_t gfp_mask)
730	{
731	void *shadow = NULL;
732	int ret;
733
734	__SetPageLocked(page);
735	ret = __add_to_page_cache_locked(page, mapping, offset,
736	gfp_mask, &shadow);
737	if (unlikely(ret))
738	__ClearPageLocked(page);
739	else {
740	/*
741	* The page might have been evicted from cache only
742	* recently, in which case it should be activated like
743	* any other repeatedly accessed page.
744	* The exception is pages getting rewritten; evicting other
745	* data from the working set, only to cache data that will
746	* get overwritten with something else, is a waste of memory.
747	*/
748	WARN_ON_ONCE(PageActive(page));
749	if (!(gfp_mask & __GFP_WRITE) && shadow)
750	workingset_refault(page, shadow);
751	lru_cache_add(page);
752	}
753	return ret;
754	}
755	EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
756
757	#ifdef CONFIG_NUMA
758	struct page *__page_cache_alloc(gfp_t gfp)
759	{
760	int n;
761	struct page *page;
762
763	if (cpuset_do_page_mem_spread()) {
764	unsigned int cpuset_mems_cookie;
765	do {
766	cpuset_mems_cookie = read_mems_allowed_begin();
767	n = cpuset_mem_spread_node();
768	page = __alloc_pages_node(n, gfp, 0);
769	} while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
770
771	return page;
772	}
773	return alloc_pages(gfp, 0);
774	}
775	EXPORT_SYMBOL(__page_cache_alloc);
776	#endif
777
778	/*
779	* In order to wait for pages to become available there must be
780	* waitqueues associated with pages. By using a hash table of
781	* waitqueues where the bucket discipline is to maintain all
782	* waiters on the same queue and wake all when any of the pages
783	* become available, and for the woken contexts to check to be
784	* sure the appropriate page became available, this saves space
785	* at a cost of "thundering herd" phenomena during rare hash
786	* collisions.
787	*/
788	#define PAGE_WAIT_TABLE_BITS 8
789	#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
790	static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
791
792	static wait_queue_head_t *page_waitqueue(struct page *page)
793	{
794	return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
795	}
796
797	void __init pagecache_init(void)
798	{
799	int i;
800
801	for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
802	init_waitqueue_head(&page_wait_table[i]);
803
804	page_writeback_init();
805	}
806
807	struct wait_page_key {
808	struct page *page;
809	int bit_nr;
810	int page_match;
811	};
812
813	struct wait_page_queue {
814	struct page *page;
815	int bit_nr;
816	wait_queue_t wait;
817	};
818
819	static int wake_page_function(wait_queue_t *wait, unsigned mode, int sync, void *arg)
820	{
821	struct wait_page_key *key = arg;
822	struct wait_page_queue *wait_page
823	= container_of(wait, struct wait_page_queue, wait);
824
825	if (wait_page->page != key->page)
826	return 0;
827	key->page_match = 1;
828
829	if (wait_page->bit_nr != key->bit_nr)
830	return 0;
831	if (test_bit(key->bit_nr, &key->page->flags))
832	return 0;
833
834	return autoremove_wake_function(wait, mode, sync, key);
835	}
836
837	void wake_up_page_bit(struct page *page, int bit_nr)
838	{
839	wait_queue_head_t *q = page_waitqueue(page);
840	struct wait_page_key key;
841	unsigned long flags;
842
843	key.page = page;
844	key.bit_nr = bit_nr;
845	key.page_match = 0;
846
847	spin_lock_irqsave(&q->lock, flags);
848	__wake_up_locked_key(q, TASK_NORMAL, &key);
849	/*
850	* It is possible for other pages to have collided on the waitqueue
851	* hash, so in that case check for a page match. That prevents a long-
852	* term waiter
853	*
854	* It is still possible to miss a case here, when we woke page waiters
855	* and removed them from the waitqueue, but there are still other
856	* page waiters.
857	*/
858	if (!waitqueue_active(q) || !key.page_match) {
859	ClearPageWaiters(page);
860	/*
861	* It's possible to miss clearing Waiters here, when we woke
862	* our page waiters, but the hashed waitqueue has waiters for
863	* other pages on it.
864	*
865	* That's okay, it's a rare case. The next waker will clear it.
866	*/
867	}
868	spin_unlock_irqrestore(&q->lock, flags);
869	}
870	EXPORT_SYMBOL(wake_up_page_bit);
871
872	static inline int wait_on_page_bit_common(wait_queue_head_t *q,
873	struct page *page, int bit_nr, int state, bool lock)
874	{
875	struct wait_page_queue wait_page;
876	wait_queue_t *wait = &wait_page.wait;
877	bool thrashing = false;
878	unsigned long pflags;
879	int ret = 0;
880
881	if (bit_nr == PG_locked &&
882	!PageUptodate(page) && PageWorkingset(page)) {
883	if (!PageSwapBacked(page))
884	delayacct_thrashing_start();
885	psi_memstall_enter(&pflags);
886	thrashing = true;
887	}
888
889	init_wait(wait);
890	wait->func = wake_page_function;
891	wait_page.page = page;
892	wait_page.bit_nr = bit_nr;
893
894	for (;;) {
895	spin_lock_irq(&q->lock);
896
897	if (likely(list_empty(&wait->task_list))) {
898	if (lock)
899	__add_wait_queue_tail_exclusive(q, wait);
900	else
901	__add_wait_queue(q, wait);
902	SetPageWaiters(page);
903	}
904
905	set_current_state(state);
906
907	spin_unlock_irq(&q->lock);
908
909	if (likely(test_bit(bit_nr, &page->flags))) {
910	io_schedule();
911	}
912
913	if (lock) {
914	if (!test_and_set_bit_lock(bit_nr, &page->flags))
915	break;
916	} else {
917	if (!test_bit(bit_nr, &page->flags))
918	break;
919	}
920
921	if (unlikely(signal_pending_state(state, current))) {
922	ret = -EINTR;
923	break;
924	}
925	}
926
927	finish_wait(q, wait);
928
929	if (thrashing) {
930	if (!PageSwapBacked(page))
931	delayacct_thrashing_end();
932	psi_memstall_leave(&pflags);
933	}
934
935	/*
936	* A signal could leave PageWaiters set. Clearing it here if
937	* !waitqueue_active would be possible (by open-coding finish_wait),
938	* but still fail to catch it in the case of wait hash collision. We
939	* already can fail to clear wait hash collision cases, so don't
940	* bother with signals either.
941	*/
942
943	return ret;
944	}
945
946	void wait_on_page_bit(struct page *page, int bit_nr)
947	{
948	wait_queue_head_t *q = page_waitqueue(page);
949	wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, false);
950	}
951	EXPORT_SYMBOL(wait_on_page_bit);
952
953	int wait_on_page_bit_killable(struct page *page, int bit_nr)
954	{
955	wait_queue_head_t *q = page_waitqueue(page);
956	return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, false);
957	}
958
959	/**
960	* add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
961	* @page: Page defining the wait queue of interest
962	* @waiter: Waiter to add to the queue
963	*
964	* Add an arbitrary @waiter to the wait queue for the nominated @page.
965	*/
966	void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
967	{
968	wait_queue_head_t *q = page_waitqueue(page);
969	unsigned long flags;
970
971	spin_lock_irqsave(&q->lock, flags);
972	__add_wait_queue(q, waiter);
973	SetPageWaiters(page);
974	spin_unlock_irqrestore(&q->lock, flags);
975	}
976	EXPORT_SYMBOL_GPL(add_page_wait_queue);
977
978	/**
979	* unlock_page - unlock a locked page
980	* @page: the page
981	*
982	* Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
983	* Also wakes sleepers in wait_on_page_writeback() because the wakeup
984	* mechanism between PageLocked pages and PageWriteback pages is shared.
985	* But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
986	*
987	* The mb is necessary to enforce ordering between the clear_bit and the read
988	* of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
989	*/
990	void unlock_page(struct page *page)
991	{
992	page = compound_head(page);
993	VM_BUG_ON_PAGE(!PageLocked(page), page);
994	clear_bit_unlock(PG_locked, &page->flags);
995	smp_mb__after_atomic();
996	wake_up_page(page, PG_locked);
997	}
998	EXPORT_SYMBOL(unlock_page);
999
1000	/**
1001	* end_page_writeback - end writeback against a page
1002	* @page: the page
1003	*/
1004	void end_page_writeback(struct page *page)
1005	{
1006	/*
1007	* TestClearPageReclaim could be used here but it is an atomic
1008	* operation and overkill in this particular case. Failing to
1009	* shuffle a page marked for immediate reclaim is too mild to
1010	* justify taking an atomic operation penalty at the end of
1011	* ever page writeback.
1012	*/
1013	if (PageReclaim(page)) {
1014	ClearPageReclaim(page);
1015	rotate_reclaimable_page(page);
1016	}
1017
1018	if (!test_clear_page_writeback(page))
1019	BUG();
1020
1021	smp_mb__after_atomic();
1022	wake_up_page(page, PG_writeback);
1023	}
1024	EXPORT_SYMBOL(end_page_writeback);
1025
1026	/*
1027	* After completing I/O on a page, call this routine to update the page
1028	* flags appropriately
1029	*/
1030	void page_endio(struct page *page, bool is_write, int err)
1031	{
1032	if (!is_write) {
1033	if (!err) {
1034	SetPageUptodate(page);
1035	} else {
1036	ClearPageUptodate(page);
1037	SetPageError(page);
1038	}
1039	unlock_page(page);
1040	} else {
1041	if (err) {
1042	struct address_space *mapping;
1043
1044	SetPageError(page);
1045	mapping = page_mapping(page);
1046	if (mapping)
1047	mapping_set_error(mapping, err);
1048	}
1049	end_page_writeback(page);
1050	}
1051	}
1052	EXPORT_SYMBOL_GPL(page_endio);
1053
1054	/**
1055	* __lock_page - get a lock on the page, assuming we need to sleep to get it
1056	* @page: the page to lock
1057	*/
1058	void __lock_page(struct page *__page)
1059	{
1060	struct page *page = compound_head(__page);
1061	wait_queue_head_t *q = page_waitqueue(page);
1062	wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, true);
1063	}
1064	EXPORT_SYMBOL(__lock_page);
1065
1066	int __lock_page_killable(struct page *__page)
1067	{
1068	struct page *page = compound_head(__page);
1069	wait_queue_head_t *q = page_waitqueue(page);
1070	return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, true);
1071	}
1072	EXPORT_SYMBOL_GPL(__lock_page_killable);
1073
1074	/*
1075	* Return values:
1076	* 1 - page is locked; mmap_sem is still held.
1077	* 0 - page is not locked.
1078	* mmap_sem has been released (up_read()), unless flags had both
1079	* FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
1080	* which case mmap_sem is still held.
1081	*
1082	* If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
1083	* with the page locked and the mmap_sem unperturbed.
1084	*/
1085	int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
1086	unsigned int flags)
1087	{
1088	if (flags & FAULT_FLAG_ALLOW_RETRY) {
1089	/*
1090	* CAUTION! In this case, mmap_sem is not released
1091	* even though return 0.
1092	*/
1093	if (flags & FAULT_FLAG_RETRY_NOWAIT)
1094	return 0;
1095
1096	up_read(&mm->mmap_sem);
1097	if (flags & FAULT_FLAG_KILLABLE)
1098	wait_on_page_locked_killable(page);
1099	else
1100	wait_on_page_locked(page);
1101	return 0;
1102	} else {
1103	if (flags & FAULT_FLAG_KILLABLE) {
1104	int ret;
1105
1106	ret = __lock_page_killable(page);
1107	if (ret) {
1108	up_read(&mm->mmap_sem);
1109	return 0;
1110	}
1111	} else
1112	__lock_page(page);
1113	return 1;
1114	}
1115	}
1116
1117	/**
1118	* page_cache_next_hole - find the next hole (not-present entry)
1119	* @mapping: mapping
1120	* @index: index
1121	* @max_scan: maximum range to search
1122	*
1123	* Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
1124	* lowest indexed hole.
1125	*
1126	* Returns: the index of the hole if found, otherwise returns an index
1127	* outside of the set specified (in which case 'return - index >=
1128	* max_scan' will be true). In rare cases of index wrap-around, 0 will
1129	* be returned.
1130	*
1131	* page_cache_next_hole may be called under rcu_read_lock. However,
1132	* like radix_tree_gang_lookup, this will not atomically search a
1133	* snapshot of the tree at a single point in time. For example, if a
1134	* hole is created at index 5, then subsequently a hole is created at
1135	* index 10, page_cache_next_hole covering both indexes may return 10
1136	* if called under rcu_read_lock.
1137	*/
1138	pgoff_t page_cache_next_hole(struct address_space *mapping,
1139	pgoff_t index, unsigned long max_scan)
1140	{
1141	unsigned long i;
1142
1143	for (i = 0; i < max_scan; i++) {
1144	struct page *page;
1145
1146	page = radix_tree_lookup(&mapping->page_tree, index);
1147	if (!page || radix_tree_exceptional_entry(page))
1148	break;
1149	index++;
1150	if (index == 0)
1151	break;
1152	}
1153
1154	return index;
1155	}
1156	EXPORT_SYMBOL(page_cache_next_hole);
1157
1158	/**
1159	* page_cache_prev_hole - find the prev hole (not-present entry)
1160	* @mapping: mapping
1161	* @index: index
1162	* @max_scan: maximum range to search
1163	*
1164	* Search backwards in the range [max(index-max_scan+1, 0), index] for
1165	* the first hole.
1166	*
1167	* Returns: the index of the hole if found, otherwise returns an index
1168	* outside of the set specified (in which case 'index - return >=
1169	* max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
1170	* will be returned.
1171	*
1172	* page_cache_prev_hole may be called under rcu_read_lock. However,
1173	* like radix_tree_gang_lookup, this will not atomically search a
1174	* snapshot of the tree at a single point in time. For example, if a
1175	* hole is created at index 10, then subsequently a hole is created at
1176	* index 5, page_cache_prev_hole covering both indexes may return 5 if
1177	* called under rcu_read_lock.
1178	*/
1179	pgoff_t page_cache_prev_hole(struct address_space *mapping,
1180	pgoff_t index, unsigned long max_scan)
1181	{
1182	unsigned long i;
1183
1184	for (i = 0; i < max_scan; i++) {
1185	struct page *page;
1186
1187	page = radix_tree_lookup(&mapping->page_tree, index);
1188	if (!page || radix_tree_exceptional_entry(page))
1189	break;
1190	index--;
1191	if (index == ULONG_MAX)
1192	break;
1193	}
1194
1195	return index;
1196	}
1197	EXPORT_SYMBOL(page_cache_prev_hole);
1198
1199	/**
1200	* find_get_entry - find and get a page cache entry
1201	* @mapping: the address_space to search
1202	* @offset: the page cache index
1203	*
1204	* Looks up the page cache slot at @mapping & @offset. If there is a
1205	* page cache page, it is returned with an increased refcount.
1206	*
1207	* If the slot holds a shadow entry of a previously evicted page, or a
1208	* swap entry from shmem/tmpfs, it is returned.
1209	*
1210	* Otherwise, %NULL is returned.
1211	*/
1212	struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
1213	{
1214	void **pagep;
1215	struct page *head, *page;
1216
1217	rcu_read_lock();
1218	repeat:
1219	page = NULL;
1220	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
1221	if (pagep) {
1222	page = radix_tree_deref_slot(pagep);
1223	if (unlikely(!page))
1224	goto out;
1225	if (radix_tree_exception(page)) {
1226	if (radix_tree_deref_retry(page))
1227	goto repeat;
1228	/*
1229	* A shadow entry of a recently evicted page,
1230	* or a swap entry from shmem/tmpfs. Return
1231	* it without attempting to raise page count.
1232	*/
1233	goto out;
1234	}
1235
1236	head = compound_head(page);
1237	if (!page_cache_get_speculative(head))
1238	goto repeat;
1239
1240	/* The page was split under us? */
1241	if (compound_head(page) != head) {
1242	put_page(head);
1243	goto repeat;
1244	}
1245
1246	/*
1247	* Has the page moved?
1248	* This is part of the lockless pagecache protocol. See
1249	* include/linux/pagemap.h for details.
1250	*/
1251	if (unlikely(page != *pagep)) {
1252	put_page(head);
1253	goto repeat;
1254	}
1255	}
1256	out:
1257	rcu_read_unlock();
1258
1259	return page;
1260	}
1261	EXPORT_SYMBOL(find_get_entry);
1262
1263	/**
1264	* find_lock_entry - locate, pin and lock a page cache entry
1265	* @mapping: the address_space to search
1266	* @offset: the page cache index
1267	*
1268	* Looks up the page cache slot at @mapping & @offset. If there is a
1269	* page cache page, it is returned locked and with an increased
1270	* refcount.
1271	*
1272	* If the slot holds a shadow entry of a previously evicted page, or a
1273	* swap entry from shmem/tmpfs, it is returned.
1274	*
1275	* Otherwise, %NULL is returned.
1276	*
1277	* find_lock_entry() may sleep.
1278	*/
1279	struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
1280	{
1281	struct page *page;
1282
1283	repeat:
1284	page = find_get_entry(mapping, offset);
1285	if (page && !radix_tree_exception(page)) {
1286	lock_page(page);
1287	/* Has the page been truncated? */
1288	if (unlikely(page_mapping(page) != mapping)) {
1289	unlock_page(page);
1290	put_page(page);
1291	goto repeat;
1292	}
1293	VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
1294	}
1295	return page;
1296	}
1297	EXPORT_SYMBOL(find_lock_entry);
1298
1299	/**
1300	* pagecache_get_page - find and get a page reference
1301	* @mapping: the address_space to search
1302	* @offset: the page index
1303	* @fgp_flags: PCG flags
1304	* @gfp_mask: gfp mask to use for the page cache data page allocation
1305	*
1306	* Looks up the page cache slot at @mapping & @offset.
1307	*
1308	* PCG flags modify how the page is returned.
1309	*
1310	* FGP_ACCESSED: the page will be marked accessed
1311	* FGP_LOCK: Page is return locked
1312	* FGP_CREAT: If page is not present then a new page is allocated using
1313	* @gfp_mask and added to the page cache and the VM's LRU
1314	* list. The page is returned locked and with an increased
1315	* refcount. Otherwise, %NULL is returned.
1316	* FGP_FOR_MMAP: Similar to FGP_CREAT, only we want to allow the caller to do
1317	* its own locking dance if the page is already in cache, or unlock the page
1318	* before returning if we had to add the page to pagecache.
1319	*
1320	* If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
1321	* if the GFP flags specified for FGP_CREAT are atomic.
1322	*
1323	* If there is a page cache page, it is returned with an increased refcount.
1324	*/
1325	struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1326	int fgp_flags, gfp_t gfp_mask)
1327	{
1328	struct page *page;
1329
1330	repeat:
1331	page = find_get_entry(mapping, offset);
1332	if (radix_tree_exceptional_entry(page))
1333	page = NULL;
1334	if (!page)
1335	goto no_page;
1336
1337	if (fgp_flags & FGP_LOCK) {
1338	if (fgp_flags & FGP_NOWAIT) {
1339	if (!trylock_page(page)) {
1340	put_page(page);
1341	return NULL;
1342	}
1343	} else {
1344	lock_page(page);
1345	}
1346
1347	/* Has the page been truncated? */
1348	if (unlikely(page->mapping != mapping)) {
1349	unlock_page(page);
1350	put_page(page);
1351	goto repeat;
1352	}
1353	VM_BUG_ON_PAGE(page->index != offset, page);
1354	}
1355
1356	if (page && (fgp_flags & FGP_ACCESSED))
1357	mark_page_accessed(page);
1358
1359	no_page:
1360	if (!page && (fgp_flags & FGP_CREAT)) {
1361	int err;
1362	if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1363	gfp_mask |= __GFP_WRITE;
1364	if (fgp_flags & FGP_NOFS)
1365	gfp_mask &= ~__GFP_FS;
1366
1367	page = __page_cache_alloc(gfp_mask);
1368	if (!page)
1369	return NULL;
1370
1371	if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
1372	fgp_flags |= FGP_LOCK;
1373
1374	/* Init accessed so avoid atomic mark_page_accessed later */
1375	if (fgp_flags & FGP_ACCESSED)
1376	__SetPageReferenced(page);
1377
1378	err = add_to_page_cache_lru(page, mapping, offset, gfp_mask);
1379	if (unlikely(err)) {
1380	put_page(page);
1381	page = NULL;
1382	if (err == -EEXIST)
1383	goto repeat;
1384	}
1385
1386	/*
1387	* add_to_page_cache_lru lock's the page, and for mmap we expect
1388	* a unlocked page.
1389	*/
1390	if (page && (fgp_flags & FGP_FOR_MMAP))
1391	unlock_page(page);
1392
1393	}
1394
1395	return page;
1396	}
1397	EXPORT_SYMBOL(pagecache_get_page);
1398
1399	/**
1400	* find_get_entries - gang pagecache lookup
1401	* @mapping: The address_space to search
1402	* @start: The starting page cache index
1403	* @nr_entries: The maximum number of entries
1404	* @entries: Where the resulting entries are placed
1405	* @indices: The cache indices corresponding to the entries in @entries
1406	*
1407	* find_get_entries() will search for and return a group of up to
1408	* @nr_entries entries in the mapping. The entries are placed at
1409	* @entries. find_get_entries() takes a reference against any actual
1410	* pages it returns.
1411	*
1412	* The search returns a group of mapping-contiguous page cache entries
1413	* with ascending indexes. There may be holes in the indices due to
1414	* not-present pages.
1415	*
1416	* Any shadow entries of evicted pages, or swap entries from
1417	* shmem/tmpfs, are included in the returned array.
1418	*
1419	* find_get_entries() returns the number of pages and shadow entries
1420	* which were found.
1421	*/
1422	unsigned find_get_entries(struct address_space *mapping,
1423	pgoff_t start, unsigned int nr_entries,
1424	struct page **entries, pgoff_t *indices)
1425	{
1426	void **slot;
1427	unsigned int ret = 0;
1428	struct radix_tree_iter iter;
1429
1430	if (!nr_entries)
1431	return 0;
1432
1433	rcu_read_lock();
1434	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1435	struct page *head, *page;
1436	repeat:
1437	page = radix_tree_deref_slot(slot);
1438	if (unlikely(!page))
1439	continue;
1440	if (radix_tree_exception(page)) {
1441	if (radix_tree_deref_retry(page)) {
1442	slot = radix_tree_iter_retry(&iter);
1443	continue;
1444	}
1445	/*
1446	* A shadow entry of a recently evicted page, a swap
1447	* entry from shmem/tmpfs or a DAX entry. Return it
1448	* without attempting to raise page count.
1449	*/
1450	goto export;
1451	}
1452
1453	head = compound_head(page);
1454	if (!page_cache_get_speculative(head))
1455	goto repeat;
1456
1457	/* The page was split under us? */
1458	if (compound_head(page) != head) {
1459	put_page(head);
1460	goto repeat;
1461	}
1462
1463	/* Has the page moved? */
1464	if (unlikely(page != *slot)) {
1465	put_page(head);
1466	goto repeat;
1467	}
1468	export:
1469	indices[ret] = iter.index;
1470	entries[ret] = page;
1471	if (++ret == nr_entries)
1472	break;
1473	}
1474	rcu_read_unlock();
1475	return ret;
1476	}
1477
1478	/**
1479	* find_get_pages - gang pagecache lookup
1480	* @mapping: The address_space to search
1481	* @start: The starting page index
1482	* @nr_pages: The maximum number of pages
1483	* @pages: Where the resulting pages are placed
1484	*
1485	* find_get_pages() will search for and return a group of up to
1486	* @nr_pages pages in the mapping. The pages are placed at @pages.
1487	* find_get_pages() takes a reference against the returned pages.
1488	*
1489	* The search returns a group of mapping-contiguous pages with ascending
1490	* indexes. There may be holes in the indices due to not-present pages.
1491	*
1492	* find_get_pages() returns the number of pages which were found.
1493	*/
1494	unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
1495	unsigned int nr_pages, struct page **pages)
1496	{
1497	struct radix_tree_iter iter;
1498	void **slot;
1499	unsigned ret = 0;
1500
1501	if (unlikely(!nr_pages))
1502	return 0;
1503
1504	rcu_read_lock();
1505	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1506	struct page *head, *page;
1507	repeat:
1508	page = radix_tree_deref_slot(slot);
1509	if (unlikely(!page))
1510	continue;
1511
1512	if (radix_tree_exception(page)) {
1513	if (radix_tree_deref_retry(page)) {
1514	slot = radix_tree_iter_retry(&iter);
1515	continue;
1516	}
1517	/*
1518	* A shadow entry of a recently evicted page,
1519	* or a swap entry from shmem/tmpfs. Skip
1520	* over it.
1521	*/
1522	continue;
1523	}
1524
1525	head = compound_head(page);
1526	if (!page_cache_get_speculative(head))
1527	goto repeat;
1528
1529	/* The page was split under us? */
1530	if (compound_head(page) != head) {
1531	put_page(head);
1532	goto repeat;
1533	}
1534
1535	/* Has the page moved? */
1536	if (unlikely(page != *slot)) {
1537	put_page(head);
1538	goto repeat;
1539	}
1540
1541	pages[ret] = page;
1542	if (++ret == nr_pages)
1543	break;
1544	}
1545
1546	rcu_read_unlock();
1547	return ret;
1548	}
1549
1550	/**
1551	* find_get_pages_contig - gang contiguous pagecache lookup
1552	* @mapping: The address_space to search
1553	* @index: The starting page index
1554	* @nr_pages: The maximum number of pages
1555	* @pages: Where the resulting pages are placed
1556	*
1557	* find_get_pages_contig() works exactly like find_get_pages(), except
1558	* that the returned number of pages are guaranteed to be contiguous.
1559	*
1560	* find_get_pages_contig() returns the number of pages which were found.
1561	*/
1562	unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1563	unsigned int nr_pages, struct page **pages)
1564	{
1565	struct radix_tree_iter iter;
1566	void **slot;
1567	unsigned int ret = 0;
1568
1569	if (unlikely(!nr_pages))
1570	return 0;
1571
1572	rcu_read_lock();
1573	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
1574	struct page *head, *page;
1575	repeat:
1576	page = radix_tree_deref_slot(slot);
1577	/* The hole, there no reason to continue */
1578	if (unlikely(!page))
1579	break;
1580
1581	if (radix_tree_exception(page)) {
1582	if (radix_tree_deref_retry(page)) {
1583	slot = radix_tree_iter_retry(&iter);
1584	continue;
1585	}
1586	/*
1587	* A shadow entry of a recently evicted page,
1588	* or a swap entry from shmem/tmpfs. Stop
1589	* looking for contiguous pages.
1590	*/
1591	break;
1592	}
1593
1594	head = compound_head(page);
1595	if (!page_cache_get_speculative(head))
1596	goto repeat;
1597
1598	/* The page was split under us? */
1599	if (compound_head(page) != head) {
1600	put_page(head);
1601	goto repeat;
1602	}
1603
1604	/* Has the page moved? */
1605	if (unlikely(page != *slot)) {
1606	put_page(head);
1607	goto repeat;
1608	}
1609
1610	/*
1611	* must check mapping and index after taking the ref.
1612	* otherwise we can get both false positives and false
1613	* negatives, which is just confusing to the caller.
1614	*/
1615	if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
1616	put_page(page);
1617	break;
1618	}
1619
1620	pages[ret] = page;
1621	if (++ret == nr_pages)
1622	break;
1623	}
1624	rcu_read_unlock();
1625	return ret;
1626	}
1627	EXPORT_SYMBOL(find_get_pages_contig);
1628
1629	/**
1630	* find_get_pages_range_tag - find and return pages in given range matching @tag
1631	* @mapping: the address_space to search
1632	* @index: the starting page index
1633	* @end: The final page index (inclusive)
1634	* @tag: the tag index
1635	* @nr_pages: the maximum number of pages
1636	* @pages: where the resulting pages are placed
1637	*
1638	* Like find_get_pages, except we only return pages which are tagged with
1639	* @tag. We update @index to index the next page for the traversal.
1640	*/
1641	unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
1642	pgoff_t end, int tag, unsigned int nr_pages,
1643	struct page **pages)
1644	{
1645	struct radix_tree_iter iter;
1646	void **slot;
1647	unsigned ret = 0;
1648
1649	if (unlikely(!nr_pages))
1650	return 0;
1651
1652	rcu_read_lock();
1653	radix_tree_for_each_tagged(slot, &mapping->page_tree,
1654	&iter, *index, tag) {
1655	struct page *head, *page;
1656
1657	if (iter.index > end)
1658	break;
1659	repeat:
1660	page = radix_tree_deref_slot(slot);
1661	if (unlikely(!page))
1662	continue;
1663
1664	if (radix_tree_exception(page)) {
1665	if (radix_tree_deref_retry(page)) {
1666	slot = radix_tree_iter_retry(&iter);
1667	continue;
1668	}
1669	/*
1670	* A shadow entry of a recently evicted page.
1671	*
1672	* Those entries should never be tagged, but
1673	* this tree walk is lockless and the tags are
1674	* looked up in bulk, one radix tree node at a
1675	* time, so there is a sizable window for page
1676	* reclaim to evict a page we saw tagged.
1677	*
1678	* Skip over it.
1679	*/
1680	continue;
1681	}
1682
1683	head = compound_head(page);
1684	if (!page_cache_get_speculative(head))
1685	goto repeat;
1686
1687	/* The page was split under us? */
1688	if (compound_head(page) != head) {
1689	put_page(head);
1690	goto repeat;
1691	}
1692
1693	/* Has the page moved? */
1694	if (unlikely(page != *slot)) {
1695	put_page(head);
1696	goto repeat;
1697	}
1698
1699	pages[ret] = page;
1700	if (++ret == nr_pages) {
1701	*index = pages[ret - 1]->index + 1;
1702	goto out;
1703	}
1704	}
1705
1706	/*
1707	* We come here when we got at @end. We take care to not overflow the
1708	* index @index as it confuses some of the callers. This breaks the
1709	* iteration when there is page at index -1 but that is already broken
1710	* anyway.
1711	*/
1712	if (end == (pgoff_t)-1)
1713	*index = (pgoff_t)-1;
1714	else
1715	*index = end + 1;
1716	out:
1717	rcu_read_unlock();
1718
1719	return ret;
1720	}
1721	EXPORT_SYMBOL(find_get_pages_range_tag);
1722
1723	/**
1724	* find_get_entries_tag - find and return entries that match @tag
1725	* @mapping: the address_space to search
1726	* @start: the starting page cache index
1727	* @tag: the tag index
1728	* @nr_entries: the maximum number of entries
1729	* @entries: where the resulting entries are placed
1730	* @indices: the cache indices corresponding to the entries in @entries
1731	*
1732	* Like find_get_entries, except we only return entries which are tagged with
1733	* @tag.
1734	*/
1735	unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1736	int tag, unsigned int nr_entries,
1737	struct page **entries, pgoff_t *indices)
1738	{
1739	void **slot;
1740	unsigned int ret = 0;
1741	struct radix_tree_iter iter;
1742
1743	if (!nr_entries)
1744	return 0;
1745
1746	rcu_read_lock();
1747	radix_tree_for_each_tagged(slot, &mapping->page_tree,
1748	&iter, start, tag) {
1749	struct page *head, *page;
1750	repeat:
1751	page = radix_tree_deref_slot(slot);
1752	if (unlikely(!page))
1753	continue;
1754	if (radix_tree_exception(page)) {
1755	if (radix_tree_deref_retry(page)) {
1756	slot = radix_tree_iter_retry(&iter);
1757	continue;
1758	}
1759
1760	/*
1761	* A shadow entry of a recently evicted page, a swap
1762	* entry from shmem/tmpfs or a DAX entry. Return it
1763	* without attempting to raise page count.
1764	*/
1765	goto export;
1766	}
1767
1768	head = compound_head(page);
1769	if (!page_cache_get_speculative(head))
1770	goto repeat;
1771
1772	/* The page was split under us? */
1773	if (compound_head(page) != head) {
1774	put_page(head);
1775	goto repeat;
1776	}
1777
1778	/* Has the page moved? */
1779	if (unlikely(page != *slot)) {
1780	put_page(head);
1781	goto repeat;
1782	}
1783	export:
1784	indices[ret] = iter.index;
1785	entries[ret] = page;
1786	if (++ret == nr_entries)
1787	break;
1788	}
1789	rcu_read_unlock();
1790	return ret;
1791	}
1792	EXPORT_SYMBOL(find_get_entries_tag);
1793
1794	/*
1795	* CD/DVDs are error prone. When a medium error occurs, the driver may fail
1796	* a _large_ part of the i/o request. Imagine the worst scenario:
1797	*
1798	* ---R__________________________________________B__________
1799	* ^ reading here ^ bad block(assume 4k)
1800	*
1801	* read(R) => miss => readahead(R...B) => media error => frustrating retries
1802	* => failing the whole request => read(R) => read(R+1) =>
1803	* readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
1804	* readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
1805	* readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
1806	*
1807	* It is going insane. Fix it by quickly scaling down the readahead size.
1808	*/
1809	static void shrink_readahead_size_eio(struct file *filp,
1810	struct file_ra_state *ra)
1811	{
1812	ra->ra_pages /= 4;
1813	}
1814
1815	/**
1816	* do_generic_file_read - generic file read routine
1817	* @filp: the file to read
1818	* @ppos: current file position
1819	* @iter: data destination
1820	* @written: already copied
1821	*
1822	* This is a generic file read routine, and uses the
1823	* mapping->a_ops->readpage() function for the actual low-level stuff.
1824	*
1825	* This is really ugly. But the goto's actually try to clarify some
1826	* of the logic when it comes to error handling etc.
1827	*/
1828	static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
1829	struct iov_iter *iter, ssize_t written)
1830	{
1831	struct address_space *mapping = filp->f_mapping;
1832	struct inode *inode = mapping->host;
1833	struct file_ra_state *ra = &filp->f_ra;
1834	pgoff_t index;
1835	pgoff_t last_index;
1836	pgoff_t prev_index;
1837	unsigned long offset; /* offset into pagecache page */
1838	unsigned int prev_offset;
1839	int error = 0;
1840
1841	if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
1842	return 0;
1843	iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
1844
1845	index = *ppos >> PAGE_SHIFT;
1846	prev_index = ra->prev_pos >> PAGE_SHIFT;
1847	prev_offset = ra->prev_pos & (PAGE_SIZE-1);
1848	last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
1849	offset = *ppos & ~PAGE_MASK;
1850
1851	for (;;) {
1852	struct page *page;
1853	pgoff_t end_index;
1854	loff_t isize;
1855	unsigned long nr, ret;
1856
1857	cond_resched();
1858	find_page:
1859	if (fatal_signal_pending(current)) {
1860	error = -EINTR;
1861	goto out;
1862	}
1863
1864	page = find_get_page(mapping, index);
1865	if (!page) {
1866	page_cache_sync_readahead(mapping,
1867	ra, filp,
1868	index, last_index - index);
1869	page = find_get_page(mapping, index);
1870	if (unlikely(page == NULL))
1871	goto no_cached_page;
1872	}
1873	if (PageReadahead(page)) {
1874	page_cache_async_readahead(mapping,
1875	ra, filp, page,
1876	index, last_index - index);
1877	}
1878	if (!PageUptodate(page)) {
1879	/*
1880	* See comment in do_read_cache_page on why
1881	* wait_on_page_locked is used to avoid unnecessarily
1882	* serialisations and why it's safe.
1883	*/
1884	error = wait_on_page_locked_killable(page);
1885	if (unlikely(error))
1886	goto readpage_error;
1887	if (PageUptodate(page))
1888	goto page_ok;
1889
1890	if (inode->i_blkbits == PAGE_SHIFT ||
1891	!mapping->a_ops->is_partially_uptodate)
1892	goto page_not_up_to_date;
1893	/* pipes can't handle partially uptodate pages */
1894	if (unlikely(iter->type & ITER_PIPE))
1895	goto page_not_up_to_date;
1896	if (!trylock_page(page))
1897	goto page_not_up_to_date;
1898	/* Did it get truncated before we got the lock? */
1899	if (!page->mapping)
1900	goto page_not_up_to_date_locked;
1901	if (!mapping->a_ops->is_partially_uptodate(page,
1902	offset, iter->count))
1903	goto page_not_up_to_date_locked;
1904	unlock_page(page);
1905	}
1906	page_ok:
1907	/*
1908	* i_size must be checked after we know the page is Uptodate.
1909	*
1910	* Checking i_size after the check allows us to calculate
1911	* the correct value for "nr", which means the zero-filled
1912	* part of the page is not copied back to userspace (unless
1913	* another truncate extends the file - this is desired though).
1914	*/
1915
1916	isize = i_size_read(inode);
1917	end_index = (isize - 1) >> PAGE_SHIFT;
1918	if (unlikely(!isize || index > end_index)) {
1919	put_page(page);
1920	goto out;
1921	}
1922
1923	/* nr is the maximum number of bytes to copy from this page */
1924	nr = PAGE_SIZE;
1925	if (index == end_index) {
1926	nr = ((isize - 1) & ~PAGE_MASK) + 1;
1927	if (nr <= offset) {
1928	put_page(page);
1929	goto out;
1930	}
1931	}
1932	nr = nr - offset;
1933
1934	/* If users can be writing to this page using arbitrary
1935	* virtual addresses, take care about potential aliasing
1936	* before reading the page on the kernel side.
1937	*/
1938	if (mapping_writably_mapped(mapping))
1939	flush_dcache_page(page);
1940
1941	/*
1942	* When a sequential read accesses a page several times,
1943	* only mark it as accessed the first time.
1944	*/
1945	if (prev_index != index || offset != prev_offset)
1946	mark_page_accessed(page);
1947	prev_index = index;
1948
1949	/*
1950	* Ok, we have the page, and it's up-to-date, so
1951	* now we can copy it to user space...
1952	*/
1953
1954	ret = copy_page_to_iter(page, offset, nr, iter);
1955	offset += ret;
1956	index += offset >> PAGE_SHIFT;
1957	offset &= ~PAGE_MASK;
1958	prev_offset = offset;
1959
1960	put_page(page);
1961	written += ret;
1962	if (!iov_iter_count(iter))
1963	goto out;
1964	if (ret < nr) {
1965	error = -EFAULT;
1966	goto out;
1967	}
1968	continue;
1969
1970	page_not_up_to_date:
1971	/* Get exclusive access to the page ... */
1972	error = lock_page_killable(page);
1973	if (unlikely(error))
1974	goto readpage_error;
1975
1976	page_not_up_to_date_locked:
1977	/* Did it get truncated before we got the lock? */
1978	if (!page->mapping) {
1979	unlock_page(page);
1980	put_page(page);
1981	continue;
1982	}
1983
1984	/* Did somebody else fill it already? */
1985	if (PageUptodate(page)) {
1986	unlock_page(page);
1987	goto page_ok;
1988	}
1989
1990	readpage:
1991	/*
1992	* A previous I/O error may have been due to temporary
1993	* failures, eg. multipath errors.
1994	* PG_error will be set again if readpage fails.
1995	*/
1996	ClearPageError(page);
1997	/* Start the actual read. The read will unlock the page. */
1998	error = mapping->a_ops->readpage(filp, page);
1999
2000	if (unlikely(error)) {
2001	if (error == AOP_TRUNCATED_PAGE) {
2002	put_page(page);
2003	error = 0;
2004	goto find_page;
2005	}
2006	goto readpage_error;
2007	}
2008
2009	if (!PageUptodate(page)) {
2010	error = lock_page_killable(page);
2011	if (unlikely(error))
2012	goto readpage_error;
2013	if (!PageUptodate(page)) {
2014	if (page->mapping == NULL) {
2015	/*
2016	* invalidate_mapping_pages got it
2017	*/
2018	unlock_page(page);
2019	put_page(page);
2020	goto find_page;
2021	}
2022	unlock_page(page);
2023	shrink_readahead_size_eio(filp, ra);
2024	error = -EIO;
2025	goto readpage_error;
2026	}
2027	unlock_page(page);
2028	}
2029
2030	goto page_ok;
2031
2032	readpage_error:
2033	/* UHHUH! A synchronous read error occurred. Report it */
2034	put_page(page);
2035	goto out;
2036
2037	no_cached_page:
2038	/*
2039	* Ok, it wasn't cached, so we need to create a new
2040	* page..
2041	*/
2042	page = page_cache_alloc_cold(mapping);
2043	if (!page) {
2044	error = -ENOMEM;
2045	goto out;
2046	}
2047	error = add_to_page_cache_lru(page, mapping, index,
2048	mapping_gfp_constraint(mapping, GFP_KERNEL));
2049	if (error) {
2050	put_page(page);
2051	if (error == -EEXIST) {
2052	error = 0;
2053	goto find_page;
2054	}
2055	goto out;
2056	}
2057	goto readpage;
2058	}
2059
2060	out:
2061	ra->prev_pos = prev_index;
2062	ra->prev_pos <<= PAGE_SHIFT;
2063	ra->prev_pos |= prev_offset;
2064
2065	*ppos = ((loff_t)index << PAGE_SHIFT) + offset;
2066	file_accessed(filp);
2067	return written ? written : error;
2068	}
2069
2070	/**
2071	* generic_file_read_iter - generic filesystem read routine
2072	* @iocb: kernel I/O control block
2073	* @iter: destination for the data read
2074	*
2075	* This is the "read_iter()" routine for all filesystems
2076	* that can use the page cache directly.
2077	*/
2078	ssize_t
2079	generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2080	{
2081	struct file *file = iocb->ki_filp;
2082	ssize_t retval = 0;
2083	size_t count = iov_iter_count(iter);
2084
2085	if (!count)
2086	goto out; /* skip atime */
2087
2088	if (iocb->ki_flags & IOCB_DIRECT) {
2089	struct address_space *mapping = file->f_mapping;
2090	struct inode *inode = mapping->host;
2091	struct iov_iter data = *iter;
2092	loff_t size;
2093
2094	size = i_size_read(inode);
2095	retval = filemap_write_and_wait_range(mapping, iocb->ki_pos,
2096	iocb->ki_pos + count - 1);
2097	if (retval < 0)
2098	goto out;
2099
2100	file_accessed(file);
2101
2102	retval = mapping->a_ops->direct_IO(iocb, &data);
2103	if (retval >= 0) {
2104	iocb->ki_pos += retval;
2105	iov_iter_advance(iter, retval);
2106	}
2107
2108	/*
2109	* Btrfs can have a short DIO read if we encounter
2110	* compressed extents, so if there was an error, or if
2111	* we've already read everything we wanted to, or if
2112	* there was a short read because we hit EOF, go ahead
2113	* and return. Otherwise fallthrough to buffered io for
2114	* the rest of the read. Buffered reads will not work for
2115	* DAX files, so don't bother trying.
2116	*/
2117	if (retval < 0 || !iov_iter_count(iter) || iocb->ki_pos >= size ||
2118	IS_DAX(inode))
2119	goto out;
2120	}
2121
2122	retval = do_generic_file_read(file, &iocb->ki_pos, iter, retval);
2123	out:
2124	return retval;
2125	}
2126	EXPORT_SYMBOL(generic_file_read_iter);
2127
2128	#ifdef CONFIG_MMU
2129	#define MMAP_LOTSAMISS (100)
2130
2131	static struct file *maybe_unlock_mmap_for_io(struct vm_area_struct *vma,
2132	unsigned long flags, struct file *fpin)
2133	{
2134	if (fpin)
2135	return fpin;
2136
2137	/*
2138	* FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
2139	* anything, so we only pin the file and drop the mmap_sem if only
2140	* FAULT_FLAG_ALLOW_RETRY is set.
2141	*/
2142	if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) ==
2143	FAULT_FLAG_ALLOW_RETRY) {
2144	fpin = get_file(vma->vm_file);
2145	up_read(&vma->vm_mm->mmap_sem);
2146	}
2147	return fpin;
2148	}
2149
2150	/*
2151	* lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem
2152	* @vmf - the vm_fault for this fault.
2153	* @page - the page to lock.
2154	* @fpin - the pointer to the file we may pin (or is already pinned).
2155	*
2156	* This works similar to lock_page_or_retry in that it can drop the mmap_sem.
2157	* It differs in that it actually returns the page locked if it returns 1 and 0
2158	* if it couldn't lock the page. If we did have to drop the mmap_sem then fpin
2159	* will point to the pinned file and needs to be fput()'ed at a later point.
2160	*/
2161	static int lock_page_maybe_drop_mmap(struct vm_area_struct *vma,
2162	unsigned long flags, struct page *page, struct file **fpin)
2163	{
2164	if (trylock_page(page))
2165	return 1;
2166
2167	/*
2168	* NOTE! This will make us return with VM_FAULT_RETRY, but with
2169	* the mmap_sem still held. That's how FAULT_FLAG_RETRY_NOWAIT
2170	* is supposed to work. We have way too many special cases..
2171	*/
2172	if (flags & FAULT_FLAG_RETRY_NOWAIT)
2173	return 0;
2174	*fpin = maybe_unlock_mmap_for_io(vma, flags, *fpin);
2175	if (flags & FAULT_FLAG_KILLABLE) {
2176	if (__lock_page_killable(page)) {
2177	/*
2178	* We didn't have the right flags to drop the mmap_sem,
2179	* but all fault_handlers only check for fatal signals
2180	* if we return VM_FAULT_RETRY, so we need to drop the
2181	* mmap_sem here and return 0 if we don't have a fpin.
2182	*/
2183	if (*fpin == NULL)
2184	up_read(&vma->vm_mm->mmap_sem);
2185	return 0;
2186	}
2187	} else
2188	__lock_page(page);
2189	return 1;
2190	}
2191
2192	/*
2193	* Synchronous readahead happens when we don't even find a page in the page
2194	* cache at all. We don't want to perform IO under the mmap sem, so if we have
2195	* to drop the mmap sem we return the file that was pinned in order for us to do
2196	* that. If we didn't pin a file then we return NULL. The file that is
2197	* returned needs to be fput()'ed when we're done with it.
2198	*/
2199	static struct file *do_sync_mmap_readahead(struct vm_area_struct *vma,
2200	unsigned long flags,
2201	struct file_ra_state *ra,
2202	struct file *file,
2203	pgoff_t offset)
2204	{
2205	struct file *fpin = NULL;
2206	struct address_space *mapping = file->f_mapping;
2207
2208	/* If we don't want any read-ahead, don't bother */
2209	if (vma->vm_flags & VM_RAND_READ)
2210	return fpin;
2211	if (!ra->ra_pages)
2212	return fpin;
2213
2214	if (vma->vm_flags & VM_SEQ_READ) {
2215	fpin = maybe_unlock_mmap_for_io(vma, flags, fpin);
2216	page_cache_sync_readahead(mapping, ra, file, offset,
2217	ra->ra_pages);
2218	return fpin;
2219	}
2220
2221	/* Avoid banging the cache line if not needed */
2222	if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
2223	ra->mmap_miss++;
2224
2225	/*
2226	* Do we miss much more than hit in this file? If so,
2227	* stop bothering with read-ahead. It will only hurt.
2228	*/
2229	if (ra->mmap_miss > MMAP_LOTSAMISS)
2230	return fpin;
2231
2232	/*
2233	* mmap read-around
2234	*/
2235	fpin = maybe_unlock_mmap_for_io(vma, flags, fpin);
2236	ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
2237	ra->size = ra->ra_pages;
2238	ra->async_size = ra->ra_pages / 4;
2239	ra_submit(ra, mapping, file);
2240	return fpin;
2241	}
2242
2243	/*
2244	* Asynchronous readahead happens when we find the page and PG_readahead,
2245	* so we want to possibly extend the readahead further. We return the file that
2246	* was pinned if we have to drop the mmap_sem in order to do IO.
2247	*/
2248	static struct file *do_async_mmap_readahead(struct vm_area_struct *vma,
2249	unsigned long flags,
2250	struct file_ra_state *ra,
2251	struct file *file,
2252	struct page *page,
2253	pgoff_t offset)
2254	{
2255	struct address_space *mapping = file->f_mapping;
2256	struct file *fpin = NULL;
2257
2258	/* If we don't want any read-ahead, don't bother */
2259	if (vma->vm_flags & VM_RAND_READ)
2260	return fpin;
2261	if (ra->mmap_miss > 0)
2262	ra->mmap_miss--;
2263	if (PageReadahead(page)) {
2264	fpin = maybe_unlock_mmap_for_io(vma, flags, fpin);
2265	page_cache_async_readahead(mapping, ra, file,
2266	page, offset, ra->ra_pages);
2267	}
2268	return fpin;
2269	}
2270
2271	/**
2272	* filemap_fault - read in file data for page fault handling
2273	* @vma: vma in which the fault was taken
2274	* @vmf: struct vm_fault containing details of the fault
2275	*
2276	* filemap_fault() is invoked via the vma operations vector for a
2277	* mapped memory region to read in file data during a page fault.
2278	*
2279	* The goto's are kind of ugly, but this streamlines the normal case of having
2280	* it in the page cache, and handles the special cases reasonably without
2281	* having a lot of duplicated code.
2282	*
2283	* vma->vm_mm->mmap_sem must be held on entry.
2284	*
2285	* If our return value has VM_FAULT_RETRY set, it's because
2286	* lock_page_or_retry() returned 0.
2287	* The mmap_sem has usually been released in this case.
2288	* See __lock_page_or_retry() for the exception.
2289	*
2290	* If our return value does not have VM_FAULT_RETRY set, the mmap_sem
2291	* has not been released.
2292	*
2293	* We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
2294	*/
2295	int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2296	{
2297	int error;
2298	struct file *file = vma->vm_file;
2299	struct file *fpin = NULL;
2300	struct address_space *mapping = file->f_mapping;
2301	struct file_ra_state *ra = &file->f_ra;
2302	struct inode *inode = mapping->host;
2303	pgoff_t offset = vmf->pgoff;
2304	struct page *page;
2305	loff_t size;
2306	int ret = 0;
2307
2308	size = round_up(i_size_read(inode), PAGE_SIZE);
2309	if (offset >= size >> PAGE_SHIFT)
2310	return VM_FAULT_SIGBUS;
2311
2312	/*
2313	* Do we have something in the page cache already?
2314	*/
2315	page = find_get_page(mapping, offset);
2316	if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
2317	/*
2318	* We found the page, so try async readahead before
2319	* waiting for the lock.
2320	*/
2321	fpin = do_async_mmap_readahead(vma, vmf->flags, ra,
2322	file, page, offset);
2323	} else if (!page) {
2324	/* No page in the page cache at all */
2325	count_vm_event(PGMAJFAULT);
2326	mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
2327	ret = VM_FAULT_MAJOR;
2328	fpin = do_sync_mmap_readahead(vma, vmf->flags, ra,
2329	file, offset);
2330	retry_find:
2331	page = pagecache_get_page(mapping, offset,
2332	FGP_CREAT|FGP_FOR_MMAP,
2333	vmf->gfp_mask);
2334	if (!page) {
2335	if (fpin)
2336	goto out_retry;
2337	return VM_FAULT_OOM;
2338	}
2339	}
2340	if (!lock_page_maybe_drop_mmap(vma, vmf->flags, page, &fpin))
2341	goto out_retry;
2342
2343	/* Did it get truncated? */
2344	if (unlikely(page->mapping != mapping)) {
2345	unlock_page(page);
2346	put_page(page);
2347	goto retry_find;
2348	}
2349	VM_BUG_ON_PAGE(page->index != offset, page);
2350
2351	/*
2352	* We have a locked page in the page cache, now we need to check
2353	* that it's up-to-date. If not, it is going to be due to an error.
2354	*/
2355	if (unlikely(!PageUptodate(page)))
2356	goto page_not_uptodate;
2357
2358	/*
2359	* We've made it this far and we had to drop our mmap_sem, now is the
2360	* time to return to the upper layer and have it re-find the vma and
2361	* redo the fault.
2362	*/
2363	if (fpin) {
2364	unlock_page(page);
2365	goto out_retry;
2366	}
2367
2368	/*
2369	* Found the page and have a reference on it.
2370	* We must recheck i_size under page lock.
2371	*/
2372	size = round_up(i_size_read(inode), PAGE_SIZE);
2373	if (unlikely(offset >= size >> PAGE_SHIFT)) {
2374	unlock_page(page);
2375	put_page(page);
2376	return VM_FAULT_SIGBUS;
2377	}
2378
2379	vmf->page = page;
2380	return ret | VM_FAULT_LOCKED;
2381
2382	page_not_uptodate:
2383	/*
2384	* Umm, take care of errors if the page isn't up-to-date.
2385	* Try to re-read it _once_. We do this synchronously,
2386	* because there really aren't any performance issues here
2387	* and we need to check for errors.
2388	*/
2389	ClearPageError(page);
2390	fpin = maybe_unlock_mmap_for_io(vma, vmf->flags, fpin);
2391	error = mapping->a_ops->readpage(file, page);
2392	if (!error) {
2393	wait_on_page_locked(page);
2394	if (!PageUptodate(page))
2395	error = -EIO;
2396	}
2397	if (fpin)
2398	goto out_retry;
2399	put_page(page);
2400
2401	if (!error || error == AOP_TRUNCATED_PAGE)
2402	goto retry_find;
2403
2404	/* Things didn't work out. Return zero to tell the mm layer so. */
2405	shrink_readahead_size_eio(file, ra);
2406	return VM_FAULT_SIGBUS;
2407
2408	out_retry:
2409	/*
2410	* We dropped the mmap_sem, we need to return to the fault handler to
2411	* re-find the vma and come back and find our hopefully still populated
2412	* page.
2413	*/
2414	if (page)
2415	put_page(page);
2416	if (fpin)
2417	fput(fpin);
2418	return ret | VM_FAULT_RETRY;
2419	}
2420	EXPORT_SYMBOL(filemap_fault);
2421
2422	void filemap_map_pages(struct fault_env *fe,
2423	pgoff_t start_pgoff, pgoff_t end_pgoff)
2424	{
2425	struct radix_tree_iter iter;
2426	void **slot;
2427	struct file *file = fe->vma->vm_file;
2428	struct address_space *mapping = file->f_mapping;
2429	pgoff_t last_pgoff = start_pgoff;
2430	loff_t size;
2431	struct page *head, *page;
2432
2433	rcu_read_lock();
2434	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter,
2435	start_pgoff) {
2436	if (iter.index > end_pgoff)
2437	break;
2438	repeat:
2439	page = radix_tree_deref_slot(slot);
2440	if (unlikely(!page))
2441	goto next;
2442	if (radix_tree_exception(page)) {
2443	if (radix_tree_deref_retry(page)) {
2444	slot = radix_tree_iter_retry(&iter);
2445	continue;
2446	}
2447	goto next;
2448	}
2449
2450	head = compound_head(page);
2451	if (!page_cache_get_speculative(head))
2452	goto repeat;
2453
2454	/* The page was split under us? */
2455	if (compound_head(page) != head) {
2456	put_page(head);
2457	goto repeat;
2458	}
2459
2460	/* Has the page moved? */
2461	if (unlikely(page != *slot)) {
2462	put_page(head);
2463	goto repeat;
2464	}
2465
2466	if (!PageUptodate(page) ||
2467	PageReadahead(page) ||
2468	PageHWPoison(page))
2469	goto skip;
2470	if (!trylock_page(page))
2471	goto skip;
2472
2473	if (page->mapping != mapping || !PageUptodate(page))
2474	goto unlock;
2475
2476	size = round_up(i_size_read(mapping->host), PAGE_SIZE);
2477	if (page->index >= size >> PAGE_SHIFT)
2478	goto unlock;
2479
2480	if (file->f_ra.mmap_miss > 0)
2481	file->f_ra.mmap_miss--;
2482
2483	fe->address += (iter.index - last_pgoff) << PAGE_SHIFT;
2484	if (fe->pte)
2485	fe->pte += iter.index - last_pgoff;
2486	last_pgoff = iter.index;
2487	if (alloc_set_pte(fe, NULL, page))
2488	goto unlock;
2489	unlock_page(page);
2490	goto next;
2491	unlock:
2492	unlock_page(page);
2493	skip:
2494	put_page(page);
2495	next:
2496	/* Huge page is mapped? No need to proceed. */
2497	if (pmd_trans_huge(*fe->pmd))
2498	break;
2499	if (iter.index == end_pgoff)
2500	break;
2501	}
2502	rcu_read_unlock();
2503	}
2504	EXPORT_SYMBOL(filemap_map_pages);
2505
2506	int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
2507	{
2508	struct page *page = vmf->page;
2509	struct inode *inode = file_inode(vma->vm_file);
2510	int ret = VM_FAULT_LOCKED;
2511
2512	sb_start_pagefault(inode->i_sb);
2513	file_update_time(vma->vm_file);
2514	lock_page(page);
2515	if (page->mapping != inode->i_mapping) {
2516	unlock_page(page);
2517	ret = VM_FAULT_NOPAGE;
2518	goto out;
2519	}
2520	/*
2521	* We mark the page dirty already here so that when freeze is in
2522	* progress, we are guaranteed that writeback during freezing will
2523	* see the dirty page and writeprotect it again.
2524	*/
2525	set_page_dirty(page);
2526	wait_for_stable_page(page);
2527	out:
2528	sb_end_pagefault(inode->i_sb);
2529	return ret;
2530	}
2531	EXPORT_SYMBOL(filemap_page_mkwrite);
2532
2533	const struct vm_operations_struct generic_file_vm_ops = {
2534	.fault = filemap_fault,
2535	.map_pages = filemap_map_pages,
2536	.page_mkwrite = filemap_page_mkwrite,
2537	};
2538
2539	/* This is used for a general mmap of a disk file */
2540
2541	int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2542	{
2543	struct address_space *mapping = file->f_mapping;
2544
2545	if (!mapping->a_ops->readpage)
2546	return -ENOEXEC;
2547	file_accessed(file);
2548	vma->vm_ops = &generic_file_vm_ops;
2549	return 0;
2550	}
2551
2552	/*
2553	* This is for filesystems which do not implement ->writepage.
2554	*/
2555	int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
2556	{
2557	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2558	return -EINVAL;
2559	return generic_file_mmap(file, vma);
2560	}
2561	#else
2562	int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2563	{
2564	return -ENOSYS;
2565	}
2566	int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
2567	{
2568	return -ENOSYS;
2569	}
2570	#endif /* CONFIG_MMU */
2571
2572	EXPORT_SYMBOL(generic_file_mmap);
2573	EXPORT_SYMBOL(generic_file_readonly_mmap);
2574
2575	static struct page *wait_on_page_read(struct page *page)
2576	{
2577	if (!IS_ERR(page)) {
2578	wait_on_page_locked(page);
2579	if (!PageUptodate(page)) {
2580	put_page(page);
2581	page = ERR_PTR(-EIO);
2582	}
2583	}
2584	return page;
2585	}
2586
2587	static struct page *do_read_cache_page(struct address_space *mapping,
2588	pgoff_t index,
2589	int (*filler)(struct file *, struct page *),
2590	void *data,
2591	gfp_t gfp)
2592	{
2593	struct page *page;
2594	int err;
2595	repeat:
2596	page = find_get_page(mapping, index);
2597	if (!page) {
2598	page = __page_cache_alloc(gfp | __GFP_COLD);
2599	if (!page)
2600	return ERR_PTR(-ENOMEM);
2601	err = add_to_page_cache_lru(page, mapping, index, gfp);
2602	if (unlikely(err)) {
2603	put_page(page);
2604	if (err == -EEXIST)
2605	goto repeat;
2606	/* Presumably ENOMEM for radix tree node */
2607	return ERR_PTR(err);
2608	}
2609
2610	filler:
2611	err = filler(data, page);
2612	if (err < 0) {
2613	put_page(page);
2614	return ERR_PTR(err);
2615	}
2616
2617	page = wait_on_page_read(page);
2618	if (IS_ERR(page))
2619	return page;
2620	goto out;
2621	}
2622	if (PageUptodate(page))
2623	goto out;
2624
2625	/*
2626	* Page is not up to date and may be locked due one of the following
2627	* case a: Page is being filled and the page lock is held
2628	* case b: Read/write error clearing the page uptodate status
2629	* case c: Truncation in progress (page locked)
2630	* case d: Reclaim in progress
2631	*
2632	* Case a, the page will be up to date when the page is unlocked.
2633	* There is no need to serialise on the page lock here as the page
2634	* is pinned so the lock gives no additional protection. Even if the
2635	* the page is truncated, the data is still valid if PageUptodate as
2636	* it's a race vs truncate race.
2637	* Case b, the page will not be up to date
2638	* Case c, the page may be truncated but in itself, the data may still
2639	* be valid after IO completes as it's a read vs truncate race. The
2640	* operation must restart if the page is not uptodate on unlock but
2641	* otherwise serialising on page lock to stabilise the mapping gives
2642	* no additional guarantees to the caller as the page lock is
2643	* released before return.
2644	* Case d, similar to truncation. If reclaim holds the page lock, it
2645	* will be a race with remove_mapping that determines if the mapping
2646	* is valid on unlock but otherwise the data is valid and there is
2647	* no need to serialise with page lock.
2648	*
2649	* As the page lock gives no additional guarantee, we optimistically
2650	* wait on the page to be unlocked and check if it's up to date and
2651	* use the page if it is. Otherwise, the page lock is required to
2652	* distinguish between the different cases. The motivation is that we
2653	* avoid spurious serialisations and wakeups when multiple processes
2654	* wait on the same page for IO to complete.
2655	*/
2656	wait_on_page_locked(page);
2657	if (PageUptodate(page))
2658	goto out;
2659
2660	/* Distinguish between all the cases under the safety of the lock */
2661	lock_page(page);
2662
2663	/* Case c or d, restart the operation */
2664	if (!page->mapping) {
2665	unlock_page(page);
2666	put_page(page);
2667	goto repeat;
2668	}
2669
2670	/* Someone else locked and filled the page in a very small window */
2671	if (PageUptodate(page)) {
2672	unlock_page(page);
2673	goto out;
2674	}
2675	goto filler;
2676
2677	out:
2678	mark_page_accessed(page);
2679	return page;
2680	}
2681
2682	/**
2683	* read_cache_page - read into page cache, fill it if needed
2684	* @mapping: the page's address_space
2685	* @index: the page index
2686	* @filler: function to perform the read
2687	* @data: first arg to filler(data, page) function, often left as NULL
2688	*
2689	* Read into the page cache. If a page already exists, and PageUptodate() is
2690	* not set, try to fill the page and wait for it to become unlocked.
2691	*
2692	* If the page does not get brought uptodate, return -EIO.
2693	*/
2694	struct page *read_cache_page(struct address_space *mapping,
2695	pgoff_t index,
2696	int (*filler)(struct file *, struct page *),
2697	void *data)
2698	{
2699	return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
2700	}
2701	EXPORT_SYMBOL(read_cache_page);
2702
2703	/**
2704	* read_cache_page_gfp - read into page cache, using specified page allocation flags.
2705	* @mapping: the page's address_space
2706	* @index: the page index
2707	* @gfp: the page allocator flags to use if allocating
2708	*
2709	* This is the same as "read_mapping_page(mapping, index, NULL)", but with
2710	* any new page allocations done using the specified allocation flags.
2711	*
2712	* If the page does not get brought uptodate, return -EIO.
2713	*/
2714	struct page *read_cache_page_gfp(struct address_space *mapping,
2715	pgoff_t index,
2716	gfp_t gfp)
2717	{
2718	filler_t *filler = mapping->a_ops->readpage;
2719
2720	return do_read_cache_page(mapping, index, filler, NULL, gfp);
2721	}
2722	EXPORT_SYMBOL(read_cache_page_gfp);
2723
2724	/*
2725	* Performs necessary checks before doing a write
2726	*
2727	* Can adjust writing position or amount of bytes to write.
2728	* Returns appropriate error code that caller should return or
2729	* zero in case that write should be allowed.
2730	*/
2731	inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
2732	{
2733	struct file *file = iocb->ki_filp;
2734	struct inode *inode = file->f_mapping->host;
2735	unsigned long limit = rlimit(RLIMIT_FSIZE);
2736	loff_t pos;
2737
2738	if (!iov_iter_count(from))
2739	return 0;
2740
2741	/* FIXME: this is for backwards compatibility with 2.4 */
2742	if (iocb->ki_flags & IOCB_APPEND)
2743	iocb->ki_pos = i_size_read(inode);
2744
2745	pos = iocb->ki_pos;
2746
2747	if (limit != RLIM_INFINITY) {
2748	if (iocb->ki_pos >= limit) {
2749	send_sig(SIGXFSZ, current, 0);
2750	return -EFBIG;
2751	}
2752	iov_iter_truncate(from, limit - (unsigned long)pos);
2753	}
2754
2755	/*
2756	* LFS rule
2757	*/
2758	if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
2759	!(file->f_flags & O_LARGEFILE))) {
2760	if (pos >= MAX_NON_LFS)
2761	return -EFBIG;
2762	iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
2763	}
2764
2765	/*
2766	* Are we about to exceed the fs block limit ?
2767	*
2768	* If we have written data it becomes a short write. If we have
2769	* exceeded without writing data we send a signal and return EFBIG.
2770	* Linus frestrict idea will clean these up nicely..
2771	*/
2772	if (unlikely(pos >= inode->i_sb->s_maxbytes))
2773	return -EFBIG;
2774
2775	iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
2776	return iov_iter_count(from);
2777	}
2778	EXPORT_SYMBOL(generic_write_checks);
2779
2780	int pagecache_write_begin(struct file *file, struct address_space *mapping,
2781	loff_t pos, unsigned len, unsigned flags,
2782	struct page **pagep, void **fsdata)
2783	{
2784	const struct address_space_operations *aops = mapping->a_ops;
2785
2786	return aops->write_begin(file, mapping, pos, len, flags,
2787	pagep, fsdata);
2788	}
2789	EXPORT_SYMBOL(pagecache_write_begin);
2790
2791	int pagecache_write_end(struct file *file, struct address_space *mapping,
2792	loff_t pos, unsigned len, unsigned copied,
2793	struct page *page, void *fsdata)
2794	{
2795	const struct address_space_operations *aops = mapping->a_ops;
2796
2797	return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
2798	}
2799	EXPORT_SYMBOL(pagecache_write_end);
2800
2801	ssize_t
2802	generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
2803	{
2804	struct file *file = iocb->ki_filp;
2805	struct address_space *mapping = file->f_mapping;
2806	struct inode *inode = mapping->host;
2807	loff_t pos = iocb->ki_pos;
2808	ssize_t written;
2809	size_t write_len;
2810	pgoff_t end;
2811	struct iov_iter data;
2812
2813	write_len = iov_iter_count(from);
2814	end = (pos + write_len - 1) >> PAGE_SHIFT;
2815
2816	written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
2817	if (written)
2818	goto out;
2819
2820	/*
2821	* After a write we want buffered reads to be sure to go to disk to get
2822	* the new data. We invalidate clean cached page from the region we're
2823	* about to write. We do this *before* the write so that we can return
2824	* without clobbering -EIOCBQUEUED from ->direct_IO().
2825	*/
2826	if (mapping->nrpages) {
2827	written = invalidate_inode_pages2_range(mapping,
2828	pos >> PAGE_SHIFT, end);
2829	/*
2830	* If a page can not be invalidated, return 0 to fall back
2831	* to buffered write.
2832	*/
2833	if (written) {
2834	if (written == -EBUSY)
2835	return 0;
2836	goto out;
2837	}
2838	}
2839
2840	data = *from;
2841	written = mapping->a_ops->direct_IO(iocb, &data);
2842
2843	/*
2844	* Finally, try again to invalidate clean pages which might have been
2845	* cached by non-direct readahead, or faulted in by get_user_pages()
2846	* if the source of the write was an mmap'ed region of the file
2847	* we're writing. Either one is a pretty crazy thing to do,
2848	* so we don't support it 100%. If this invalidation
2849	* fails, tough, the write still worked...
2850	*/
2851	if (mapping->nrpages) {
2852	invalidate_inode_pages2_range(mapping,
2853	pos >> PAGE_SHIFT, end);
2854	}
2855
2856	if (written > 0) {
2857	pos += written;
2858	iov_iter_advance(from, written);
2859	if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
2860	i_size_write(inode, pos);
2861	mark_inode_dirty(inode);
2862	}
2863	iocb->ki_pos = pos;
2864	}
2865	out:
2866	return written;
2867	}
2868	EXPORT_SYMBOL(generic_file_direct_write);
2869
2870	/*
2871	* Find or create a page at the given pagecache position. Return the locked
2872	* page. This function is specifically for buffered writes.
2873	*/
2874	struct page *grab_cache_page_write_begin(struct address_space *mapping,
2875	pgoff_t index, unsigned flags)
2876	{
2877	struct page *page;
2878	int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
2879
2880	if (flags & AOP_FLAG_NOFS)
2881	fgp_flags |= FGP_NOFS;
2882
2883	page = pagecache_get_page(mapping, index, fgp_flags,
2884	mapping_gfp_mask(mapping));
2885	if (page)
2886	wait_for_stable_page(page);
2887
2888	return page;
2889	}
2890	EXPORT_SYMBOL(grab_cache_page_write_begin);
2891
2892	ssize_t generic_perform_write(struct file *file,
2893	struct iov_iter *i, loff_t pos)
2894	{
2895	struct address_space *mapping = file->f_mapping;
2896	const struct address_space_operations *a_ops = mapping->a_ops;
2897	long status = 0;
2898	ssize_t written = 0;
2899	unsigned int flags = 0;
2900
2901	/*
2902	* Copies from kernel address space cannot fail (NFSD is a big user).
2903	*/
2904	if (!iter_is_iovec(i))
2905	flags |= AOP_FLAG_UNINTERRUPTIBLE;
2906
2907	do {
2908	struct page *page;
2909	unsigned long offset; /* Offset into pagecache page */
2910	unsigned long bytes; /* Bytes to write to page */
2911	size_t copied; /* Bytes copied from user */
2912	void *fsdata;
2913
2914	offset = (pos & (PAGE_SIZE - 1));
2915	bytes = min_t(unsigned long, PAGE_SIZE - offset,
2916	iov_iter_count(i));
2917
2918	again:
2919	/*
2920	* Bring in the user page that we will copy from _first_.
2921	* Otherwise there's a nasty deadlock on copying from the
2922	* same page as we're writing to, without it being marked
2923	* up-to-date.
2924	*
2925	* Not only is this an optimisation, but it is also required
2926	* to check that the address is actually valid, when atomic
2927	* usercopies are used, below.
2928	*/
2929	if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
2930	status = -EFAULT;
2931	break;
2932	}
2933
2934	if (fatal_signal_pending(current)) {
2935	status = -EINTR;
2936	break;
2937	}
2938
2939	status = a_ops->write_begin(file, mapping, pos, bytes, flags,
2940	&page, &fsdata);
2941	if (unlikely(status < 0))
2942	break;
2943
2944	if (mapping_writably_mapped(mapping))
2945	flush_dcache_page(page);
2946
2947	copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
2948	flush_dcache_page(page);
2949
2950	status = a_ops->write_end(file, mapping, pos, bytes, copied,
2951	page, fsdata);
2952	if (unlikely(status < 0))
2953	break;
2954	copied = status;
2955
2956	cond_resched();
2957
2958	iov_iter_advance(i, copied);
2959	if (unlikely(copied == 0)) {
2960	/*
2961	* If we were unable to copy any data at all, we must
2962	* fall back to a single segment length write.
2963	*
2964	* If we didn't fallback here, we could livelock
2965	* because not all segments in the iov can be copied at
2966	* once without a pagefault.
2967	*/
2968	bytes = min_t(unsigned long, PAGE_SIZE - offset,
2969	iov_iter_single_seg_count(i));
2970	goto again;
2971	}
2972	pos += copied;
2973	written += copied;
2974
2975	balance_dirty_pages_ratelimited(mapping);
2976	} while (iov_iter_count(i));
2977
2978	return written ? written : status;
2979	}
2980	EXPORT_SYMBOL(generic_perform_write);
2981
2982	/**
2983	* __generic_file_write_iter - write data to a file
2984	* @iocb: IO state structure (file, offset, etc.)
2985	* @from: iov_iter with data to write
2986	*
2987	* This function does all the work needed for actually writing data to a
2988	* file. It does all basic checks, removes SUID from the file, updates
2989	* modification times and calls proper subroutines depending on whether we
2990	* do direct IO or a standard buffered write.
2991	*
2992	* It expects i_mutex to be grabbed unless we work on a block device or similar
2993	* object which does not need locking at all.
2994	*
2995	* This function does *not* take care of syncing data in case of O_SYNC write.
2996	* A caller has to handle it. This is mainly due to the fact that we want to
2997	* avoid syncing under i_mutex.
2998	*/
2999	ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3000	{
3001	struct file *file = iocb->ki_filp;
3002	struct address_space * mapping = file->f_mapping;
3003	struct inode *inode = mapping->host;
3004	ssize_t written = 0;
3005	ssize_t err;
3006	ssize_t status;
3007
3008	/* We can write back this queue in page reclaim */
3009	current->backing_dev_info = inode_to_bdi(inode);
3010	err = file_remove_privs(file);
3011	if (err)
3012	goto out;
3013
3014	err = file_update_time(file);
3015	if (err)
3016	goto out;
3017
3018	if (iocb->ki_flags & IOCB_DIRECT) {
3019	loff_t pos, endbyte;
3020
3021	written = generic_file_direct_write(iocb, from);
3022	/*
3023	* If the write stopped short of completing, fall back to
3024	* buffered writes. Some filesystems do this for writes to
3025	* holes, for example. For DAX files, a buffered write will
3026	* not succeed (even if it did, DAX does not handle dirty
3027	* page-cache pages correctly).
3028	*/
3029	if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
3030	goto out;
3031
3032	status = generic_perform_write(file, from, pos = iocb->ki_pos);
3033	/*
3034	* If generic_perform_write() returned a synchronous error
3035	* then we want to return the number of bytes which were
3036	* direct-written, or the error code if that was zero. Note
3037	* that this differs from normal direct-io semantics, which
3038	* will return -EFOO even if some bytes were written.
3039	*/
3040	if (unlikely(status < 0)) {
3041	err = status;
3042	goto out;
3043	}
3044	/*
3045	* We need to ensure that the page cache pages are written to
3046	* disk and invalidated to preserve the expected O_DIRECT
3047	* semantics.
3048	*/
3049	endbyte = pos + status - 1;
3050	err = filemap_write_and_wait_range(mapping, pos, endbyte);
3051	if (err == 0) {
3052	iocb->ki_pos = endbyte + 1;
3053	written += status;
3054	invalidate_mapping_pages(mapping,
3055	pos >> PAGE_SHIFT,
3056	endbyte >> PAGE_SHIFT);
3057	} else {
3058	/*
3059	* We don't know how much we wrote, so just return
3060	* the number of bytes which were direct-written
3061	*/
3062	}
3063	} else {
3064	written = generic_perform_write(file, from, iocb->ki_pos);
3065	if (likely(written > 0))
3066	iocb->ki_pos += written;
3067	}
3068	out:
3069	current->backing_dev_info = NULL;
3070	return written ? written : err;
3071	}
3072	EXPORT_SYMBOL(__generic_file_write_iter);
3073
3074	/**
3075	* generic_file_write_iter - write data to a file
3076	* @iocb: IO state structure
3077	* @from: iov_iter with data to write
3078	*
3079	* This is a wrapper around __generic_file_write_iter() to be used by most
3080	* filesystems. It takes care of syncing the file in case of O_SYNC file
3081	* and acquires i_mutex as needed.
3082	*/
3083	ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3084	{
3085	struct file *file = iocb->ki_filp;
3086	struct inode *inode = file->f_mapping->host;
3087	ssize_t ret;
3088
3089	inode_lock(inode);
3090	ret = generic_write_checks(iocb, from);
3091	if (ret > 0)
3092	ret = __generic_file_write_iter(iocb, from);
3093	inode_unlock(inode);
3094
3095	if (ret > 0)
3096	ret = generic_write_sync(iocb, ret);
3097	return ret;
3098	}
3099	EXPORT_SYMBOL(generic_file_write_iter);
3100
3101	/**
3102	* try_to_release_page() - release old fs-specific metadata on a page
3103	*
3104	* @page: the page which the kernel is trying to free
3105	* @gfp_mask: memory allocation flags (and I/O mode)
3106	*
3107	* The address_space is to try to release any data against the page
3108	* (presumably at page->private). If the release was successful, return `1'.
3109	* Otherwise return zero.
3110	*
3111	* This may also be called if PG_fscache is set on a page, indicating that the
3112	* page is known to the local caching routines.
3113	*
3114	* The @gfp_mask argument specifies whether I/O may be performed to release
3115	* this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
3116	*
3117	*/
3118	int try_to_release_page(struct page *page, gfp_t gfp_mask)
3119	{
3120	struct address_space * const mapping = page->mapping;
3121
3122	BUG_ON(!PageLocked(page));
3123	if (PageWriteback(page))
3124	return 0;
3125
3126	if (mapping && mapping->a_ops->releasepage)
3127	return mapping->a_ops->releasepage(page, gfp_mask);
3128	return try_to_free_buffers(page);
3129	}
3130
3131	EXPORT_SYMBOL(try_to_release_page);
3132