path: root/block/bio.c (plain)
blob: 4c18a68913deb45fc6df162a7be68f66e2635b2f

1	/*
2	* Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
3	*
4	* This program is free software; you can redistribute it and/or modify
5	* it under the terms of the GNU General Public License version 2 as
6	* published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful,
9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	* GNU General Public License for more details.
12	*
13	* You should have received a copy of the GNU General Public Licens
14	* along with this program; if not, write to the Free Software
15	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
16	*
17	*/
18	#include <linux/mm.h>
19	#include <linux/swap.h>
20	#include <linux/bio.h>
21	#include <linux/blkdev.h>
22	#include <linux/uio.h>
23	#include <linux/iocontext.h>
24	#include <linux/slab.h>
25	#include <linux/init.h>
26	#include <linux/kernel.h>
27	#include <linux/export.h>
28	#include <linux/mempool.h>
29	#include <linux/workqueue.h>
30	#include <linux/cgroup.h>
31
32	#include <trace/events/block.h>
33
34	/*
35	* Test patch to inline a certain number of bi_io_vec's inside the bio
36	* itself, to shrink a bio data allocation from two mempool calls to one
37	*/
38	#define BIO_INLINE_VECS 4
39
40	/*
41	* if you change this list, also change bvec_alloc or things will
42	* break badly! cannot be bigger than what you can fit into an
43	* unsigned short
44	*/
45	#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
46	static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
47	BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
48	};
49	#undef BV
50
51	/*
52	* fs_bio_set is the bio_set containing bio and iovec memory pools used by
53	* IO code that does not need private memory pools.
54	*/
55	struct bio_set *fs_bio_set;
56	EXPORT_SYMBOL(fs_bio_set);
57
58	/*
59	* Our slab pool management
60	*/
61	struct bio_slab {
62	struct kmem_cache *slab;
63	unsigned int slab_ref;
64	unsigned int slab_size;
65	char name[8];
66	};
67	static DEFINE_MUTEX(bio_slab_lock);
68	static struct bio_slab *bio_slabs;
69	static unsigned int bio_slab_nr, bio_slab_max;
70
71	static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
72	{
73	unsigned int sz = sizeof(struct bio) + extra_size;
74	struct kmem_cache *slab = NULL;
75	struct bio_slab *bslab, *new_bio_slabs;
76	unsigned int new_bio_slab_max;
77	unsigned int i, entry = -1;
78
79	mutex_lock(&bio_slab_lock);
80
81	i = 0;
82	while (i < bio_slab_nr) {
83	bslab = &bio_slabs[i];
84
85	if (!bslab->slab && entry == -1)
86	entry = i;
87	else if (bslab->slab_size == sz) {
88	slab = bslab->slab;
89	bslab->slab_ref++;
90	break;
91	}
92	i++;
93	}
94
95	if (slab)
96	goto out_unlock;
97
98	if (bio_slab_nr == bio_slab_max && entry == -1) {
99	new_bio_slab_max = bio_slab_max << 1;
100	new_bio_slabs = krealloc(bio_slabs,
101	new_bio_slab_max * sizeof(struct bio_slab),
102	GFP_KERNEL);
103	if (!new_bio_slabs)
104	goto out_unlock;
105	bio_slab_max = new_bio_slab_max;
106	bio_slabs = new_bio_slabs;
107	}
108	if (entry == -1)
109	entry = bio_slab_nr++;
110
111	bslab = &bio_slabs[entry];
112
113	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
114	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
115	SLAB_HWCACHE_ALIGN, NULL);
116	if (!slab)
117	goto out_unlock;
118
119	bslab->slab = slab;
120	bslab->slab_ref = 1;
121	bslab->slab_size = sz;
122	out_unlock:
123	mutex_unlock(&bio_slab_lock);
124	return slab;
125	}
126
127	static void bio_put_slab(struct bio_set *bs)
128	{
129	struct bio_slab *bslab = NULL;
130	unsigned int i;
131
132	mutex_lock(&bio_slab_lock);
133
134	for (i = 0; i < bio_slab_nr; i++) {
135	if (bs->bio_slab == bio_slabs[i].slab) {
136	bslab = &bio_slabs[i];
137	break;
138	}
139	}
140
141	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
142	goto out;
143
144	WARN_ON(!bslab->slab_ref);
145
146	if (--bslab->slab_ref)
147	goto out;
148
149	kmem_cache_destroy(bslab->slab);
150	bslab->slab = NULL;
151
152	out:
153	mutex_unlock(&bio_slab_lock);
154	}
155
156	unsigned int bvec_nr_vecs(unsigned short idx)
157	{
158	return bvec_slabs[--idx].nr_vecs;
159	}
160
161	void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
162	{
163	if (!idx)
164	return;
165	idx--;
166
167	BIO_BUG_ON(idx >= BVEC_POOL_NR);
168
169	if (idx == BVEC_POOL_MAX) {
170	mempool_free(bv, pool);
171	} else {
172	struct biovec_slab *bvs = bvec_slabs + idx;
173
174	kmem_cache_free(bvs->slab, bv);
175	}
176	}
177
178	struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
179	mempool_t *pool)
180	{
181	struct bio_vec *bvl;
182
183	/*
184	* see comment near bvec_array define!
185	*/
186	switch (nr) {
187	case 1:
188	*idx = 0;
189	break;
190	case 2 ... 4:
191	*idx = 1;
192	break;
193	case 5 ... 16:
194	*idx = 2;
195	break;
196	case 17 ... 64:
197	*idx = 3;
198	break;
199	case 65 ... 128:
200	*idx = 4;
201	break;
202	case 129 ... BIO_MAX_PAGES:
203	*idx = 5;
204	break;
205	default:
206	return NULL;
207	}
208
209	/*
210	* idx now points to the pool we want to allocate from. only the
211	* 1-vec entry pool is mempool backed.
212	*/
213	if (*idx == BVEC_POOL_MAX) {
214	fallback:
215	bvl = mempool_alloc(pool, gfp_mask);
216	} else {
217	struct biovec_slab *bvs = bvec_slabs + *idx;
218	gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
219
220	/*
221	* Make this allocation restricted and don't dump info on
222	* allocation failures, since we'll fallback to the mempool
223	* in case of failure.
224	*/
225	__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
226
227	/*
228	* Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
229	* is set, retry with the 1-entry mempool
230	*/
231	bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
232	if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
233	*idx = BVEC_POOL_MAX;
234	goto fallback;
235	}
236	}
237
238	(*idx)++;
239	return bvl;
240	}
241
242	static void __bio_free(struct bio *bio)
243	{
244	bio_disassociate_task(bio);
245
246	if (bio_integrity(bio))
247	bio_integrity_free(bio);
248	}
249
250	static void bio_free(struct bio *bio)
251	{
252	struct bio_set *bs = bio->bi_pool;
253	void *p;
254
255	__bio_free(bio);
256
257	if (bs) {
258	bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
259
260	/*
261	* If we have front padding, adjust the bio pointer before freeing
262	*/
263	p = bio;
264	p -= bs->front_pad;
265
266	mempool_free(p, bs->bio_pool);
267	} else {
268	/* Bio was allocated by bio_kmalloc() */
269	kfree(bio);
270	}
271	}
272
273	void bio_init(struct bio *bio)
274	{
275	memset(bio, 0, sizeof(*bio));
276	atomic_set(&bio->__bi_remaining, 1);
277	atomic_set(&bio->__bi_cnt, 1);
278	}
279	EXPORT_SYMBOL(bio_init);
280
281	/**
282	* bio_reset - reinitialize a bio
283	* @bio: bio to reset
284	*
285	* Description:
286	* After calling bio_reset(), @bio will be in the same state as a freshly
287	* allocated bio returned bio bio_alloc_bioset() - the only fields that are
288	* preserved are the ones that are initialized by bio_alloc_bioset(). See
289	* comment in struct bio.
290	*/
291	void bio_reset(struct bio *bio)
292	{
293	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
294
295	__bio_free(bio);
296
297	memset(bio, 0, BIO_RESET_BYTES);
298	bio->bi_flags = flags;
299	atomic_set(&bio->__bi_remaining, 1);
300	}
301	EXPORT_SYMBOL(bio_reset);
302
303	static struct bio *__bio_chain_endio(struct bio *bio)
304	{
305	struct bio *parent = bio->bi_private;
306
307	if (!parent->bi_error)
308	parent->bi_error = bio->bi_error;
309	bio_put(bio);
310	return parent;
311	}
312
313	static void bio_chain_endio(struct bio *bio)
314	{
315	bio_endio(__bio_chain_endio(bio));
316	}
317
318	/**
319	* bio_chain - chain bio completions
320	* @bio: the target bio
321	* @parent: the @bio's parent bio
322	*
323	* The caller won't have a bi_end_io called when @bio completes - instead,
324	* @parent's bi_end_io won't be called until both @parent and @bio have
325	* completed; the chained bio will also be freed when it completes.
326	*
327	* The caller must not set bi_private or bi_end_io in @bio.
328	*/
329	void bio_chain(struct bio *bio, struct bio *parent)
330	{
331	BUG_ON(bio->bi_private || bio->bi_end_io);
332
333	bio->bi_private = parent;
334	bio->bi_end_io = bio_chain_endio;
335	bio_inc_remaining(parent);
336	}
337	EXPORT_SYMBOL(bio_chain);
338
339	static void bio_alloc_rescue(struct work_struct *work)
340	{
341	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
342	struct bio *bio;
343
344	while (1) {
345	spin_lock(&bs->rescue_lock);
346	bio = bio_list_pop(&bs->rescue_list);
347	spin_unlock(&bs->rescue_lock);
348
349	if (!bio)
350	break;
351
352	generic_make_request(bio);
353	}
354	}
355
356	static void punt_bios_to_rescuer(struct bio_set *bs)
357	{
358	struct bio_list punt, nopunt;
359	struct bio *bio;
360
361	/*
362	* In order to guarantee forward progress we must punt only bios that
363	* were allocated from this bio_set; otherwise, if there was a bio on
364	* there for a stacking driver higher up in the stack, processing it
365	* could require allocating bios from this bio_set, and doing that from
366	* our own rescuer would be bad.
367	*
368	* Since bio lists are singly linked, pop them all instead of trying to
369	* remove from the middle of the list:
370	*/
371
372	bio_list_init(&punt);
373	bio_list_init(&nopunt);
374
375	while ((bio = bio_list_pop(&current->bio_list[0])))
376	bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
377	current->bio_list[0] = nopunt;
378
379	bio_list_init(&nopunt);
380	while ((bio = bio_list_pop(&current->bio_list[1])))
381	bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
382	current->bio_list[1] = nopunt;
383
384	spin_lock(&bs->rescue_lock);
385	bio_list_merge(&bs->rescue_list, &punt);
386	spin_unlock(&bs->rescue_lock);
387
388	queue_work(bs->rescue_workqueue, &bs->rescue_work);
389	}
390
391	/**
392	* bio_alloc_bioset - allocate a bio for I/O
393	* @gfp_mask: the GFP_ mask given to the slab allocator
394	* @nr_iovecs: number of iovecs to pre-allocate
395	* @bs: the bio_set to allocate from.
396	*
397	* Description:
398	* If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
399	* backed by the @bs's mempool.
400	*
401	* When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
402	* always be able to allocate a bio. This is due to the mempool guarantees.
403	* To make this work, callers must never allocate more than 1 bio at a time
404	* from this pool. Callers that need to allocate more than 1 bio must always
405	* submit the previously allocated bio for IO before attempting to allocate
406	* a new one. Failure to do so can cause deadlocks under memory pressure.
407	*
408	* Note that when running under generic_make_request() (i.e. any block
409	* driver), bios are not submitted until after you return - see the code in
410	* generic_make_request() that converts recursion into iteration, to prevent
411	* stack overflows.
412	*
413	* This would normally mean allocating multiple bios under
414	* generic_make_request() would be susceptible to deadlocks, but we have
415	* deadlock avoidance code that resubmits any blocked bios from a rescuer
416	* thread.
417	*
418	* However, we do not guarantee forward progress for allocations from other
419	* mempools. Doing multiple allocations from the same mempool under
420	* generic_make_request() should be avoided - instead, use bio_set's front_pad
421	* for per bio allocations.
422	*
423	* RETURNS:
424	* Pointer to new bio on success, NULL on failure.
425	*/
426	struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
427	{
428	gfp_t saved_gfp = gfp_mask;
429	unsigned front_pad;
430	unsigned inline_vecs;
431	struct bio_vec *bvl = NULL;
432	struct bio *bio;
433	void *p;
434
435	if (!bs) {
436	if (nr_iovecs > UIO_MAXIOV)
437	return NULL;
438
439	p = kmalloc(sizeof(struct bio) +
440	nr_iovecs * sizeof(struct bio_vec),
441	gfp_mask);
442	front_pad = 0;
443	inline_vecs = nr_iovecs;
444	} else {
445	/* should not use nobvec bioset for nr_iovecs > 0 */
446	if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))
447	return NULL;
448	/*
449	* generic_make_request() converts recursion to iteration; this
450	* means if we're running beneath it, any bios we allocate and
451	* submit will not be submitted (and thus freed) until after we
452	* return.
453	*
454	* This exposes us to a potential deadlock if we allocate
455	* multiple bios from the same bio_set() while running
456	* underneath generic_make_request(). If we were to allocate
457	* multiple bios (say a stacking block driver that was splitting
458	* bios), we would deadlock if we exhausted the mempool's
459	* reserve.
460	*
461	* We solve this, and guarantee forward progress, with a rescuer
462	* workqueue per bio_set. If we go to allocate and there are
463	* bios on current->bio_list, we first try the allocation
464	* without __GFP_DIRECT_RECLAIM; if that fails, we punt those
465	* bios we would be blocking to the rescuer workqueue before
466	* we retry with the original gfp_flags.
467	*/
468
469	if (current->bio_list &&
470	(!bio_list_empty(&current->bio_list[0]) ||
471	!bio_list_empty(&current->bio_list[1])))
472	gfp_mask &= ~__GFP_DIRECT_RECLAIM;
473
474	p = mempool_alloc(bs->bio_pool, gfp_mask);
475	if (!p && gfp_mask != saved_gfp) {
476	punt_bios_to_rescuer(bs);
477	gfp_mask = saved_gfp;
478	p = mempool_alloc(bs->bio_pool, gfp_mask);
479	}
480
481	front_pad = bs->front_pad;
482	inline_vecs = BIO_INLINE_VECS;
483	}
484
485	if (unlikely(!p))
486	return NULL;
487
488	bio = p + front_pad;
489	bio_init(bio);
490
491	if (nr_iovecs > inline_vecs) {
492	unsigned long idx = 0;
493
494	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
495	if (!bvl && gfp_mask != saved_gfp) {
496	punt_bios_to_rescuer(bs);
497	gfp_mask = saved_gfp;
498	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
499	}
500
501	if (unlikely(!bvl))
502	goto err_free;
503
504	bio->bi_flags |= idx << BVEC_POOL_OFFSET;
505	} else if (nr_iovecs) {
506	bvl = bio->bi_inline_vecs;
507	}
508
509	bio->bi_pool = bs;
510	bio->bi_max_vecs = nr_iovecs;
511	bio->bi_io_vec = bvl;
512	return bio;
513
514	err_free:
515	mempool_free(p, bs->bio_pool);
516	return NULL;
517	}
518	EXPORT_SYMBOL(bio_alloc_bioset);
519
520	void zero_fill_bio(struct bio *bio)
521	{
522	unsigned long flags;
523	struct bio_vec bv;
524	struct bvec_iter iter;
525
526	bio_for_each_segment(bv, bio, iter) {
527	char *data = bvec_kmap_irq(&bv, &flags);
528	memset(data, 0, bv.bv_len);
529	flush_dcache_page(bv.bv_page);
530	bvec_kunmap_irq(data, &flags);
531	}
532	}
533	EXPORT_SYMBOL(zero_fill_bio);
534
535	/**
536	* bio_put - release a reference to a bio
537	* @bio: bio to release reference to
538	*
539	* Description:
540	* Put a reference to a &struct bio, either one you have gotten with
541	* bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
542	**/
543	void bio_put(struct bio *bio)
544	{
545	if (!bio_flagged(bio, BIO_REFFED))
546	bio_free(bio);
547	else {
548	BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
549
550	/*
551	* last put frees it
552	*/
553	if (atomic_dec_and_test(&bio->__bi_cnt))
554	bio_free(bio);
555	}
556	}
557	EXPORT_SYMBOL(bio_put);
558
559	inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
560	{
561	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
562	blk_recount_segments(q, bio);
563
564	return bio->bi_phys_segments;
565	}
566	EXPORT_SYMBOL(bio_phys_segments);
567
568	/**
569	* __bio_clone_fast - clone a bio that shares the original bio's biovec
570	* @bio: destination bio
571	* @bio_src: bio to clone
572	*
573	* Clone a &bio. Caller will own the returned bio, but not
574	* the actual data it points to. Reference count of returned
575	* bio will be one.
576	*
577	* Caller must ensure that @bio_src is not freed before @bio.
578	*/
579	void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
580	{
581	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
582
583	/*
584	* most users will be overriding ->bi_bdev with a new target,
585	* so we don't set nor calculate new physical/hw segment counts here
586	*/
587	bio->bi_bdev = bio_src->bi_bdev;
588	bio_set_flag(bio, BIO_CLONED);
589	bio->bi_opf = bio_src->bi_opf;
590	bio->bi_iter = bio_src->bi_iter;
591	bio->bi_io_vec = bio_src->bi_io_vec;
592
593	bio_clone_blkcg_association(bio, bio_src);
594	}
595	EXPORT_SYMBOL(__bio_clone_fast);
596
597	/**
598	* bio_clone_fast - clone a bio that shares the original bio's biovec
599	* @bio: bio to clone
600	* @gfp_mask: allocation priority
601	* @bs: bio_set to allocate from
602	*
603	* Like __bio_clone_fast, only also allocates the returned bio
604	*/
605	struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
606	{
607	struct bio *b;
608
609	b = bio_alloc_bioset(gfp_mask, 0, bs);
610	if (!b)
611	return NULL;
612
613	__bio_clone_fast(b, bio);
614
615	if (bio_integrity(bio)) {
616	int ret;
617
618	ret = bio_integrity_clone(b, bio, gfp_mask);
619
620	if (ret < 0) {
621	bio_put(b);
622	return NULL;
623	}
624	}
625
626	return b;
627	}
628	EXPORT_SYMBOL(bio_clone_fast);
629
630	/**
631	* bio_clone_bioset - clone a bio
632	* @bio_src: bio to clone
633	* @gfp_mask: allocation priority
634	* @bs: bio_set to allocate from
635	*
636	* Clone bio. Caller will own the returned bio, but not the actual data it
637	* points to. Reference count of returned bio will be one.
638	*/
639	struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
640	struct bio_set *bs)
641	{
642	struct bvec_iter iter;
643	struct bio_vec bv;
644	struct bio *bio;
645
646	/*
647	* Pre immutable biovecs, __bio_clone() used to just do a memcpy from
648	* bio_src->bi_io_vec to bio->bi_io_vec.
649	*
650	* We can't do that anymore, because:
651	*
652	* - The point of cloning the biovec is to produce a bio with a biovec
653	* the caller can modify: bi_idx and bi_bvec_done should be 0.
654	*
655	* - The original bio could've had more than BIO_MAX_PAGES biovecs; if
656	* we tried to clone the whole thing bio_alloc_bioset() would fail.
657	* But the clone should succeed as long as the number of biovecs we
658	* actually need to allocate is fewer than BIO_MAX_PAGES.
659	*
660	* - Lastly, bi_vcnt should not be looked at or relied upon by code
661	* that does not own the bio - reason being drivers don't use it for
662	* iterating over the biovec anymore, so expecting it to be kept up
663	* to date (i.e. for clones that share the parent biovec) is just
664	* asking for trouble and would force extra work on
665	* __bio_clone_fast() anyways.
666	*/
667
668	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
669	if (!bio)
670	return NULL;
671	bio->bi_bdev = bio_src->bi_bdev;
672	bio->bi_opf = bio_src->bi_opf;
673	bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
674	bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
675
676	switch (bio_op(bio)) {
677	case REQ_OP_DISCARD:
678	case REQ_OP_SECURE_ERASE:
679	break;
680	case REQ_OP_WRITE_SAME:
681	bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
682	break;
683	default:
684	bio_for_each_segment(bv, bio_src, iter)
685	bio->bi_io_vec[bio->bi_vcnt++] = bv;
686	break;
687	}
688
689	if (bio_integrity(bio_src)) {
690	int ret;
691
692	ret = bio_integrity_clone(bio, bio_src, gfp_mask);
693	if (ret < 0) {
694	bio_put(bio);
695	return NULL;
696	}
697	}
698
699	bio_clone_blkcg_association(bio, bio_src);
700
701	return bio;
702	}
703	EXPORT_SYMBOL(bio_clone_bioset);
704
705	/**
706	* bio_add_pc_page - attempt to add page to bio
707	* @q: the target queue
708	* @bio: destination bio
709	* @page: page to add
710	* @len: vec entry length
711	* @offset: vec entry offset
712	*
713	* Attempt to add a page to the bio_vec maplist. This can fail for a
714	* number of reasons, such as the bio being full or target block device
715	* limitations. The target block device must allow bio's up to PAGE_SIZE,
716	* so it is always possible to add a single page to an empty bio.
717	*
718	* This should only be used by REQ_PC bios.
719	*/
720	int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
721	*page, unsigned int len, unsigned int offset)
722	{
723	int retried_segments = 0;
724	struct bio_vec *bvec;
725
726	/*
727	* cloned bio must not modify vec list
728	*/
729	if (unlikely(bio_flagged(bio, BIO_CLONED)))
730	return 0;
731
732	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
733	return 0;
734
735	/*
736	* For filesystems with a blocksize smaller than the pagesize
737	* we will often be called with the same page as last time and
738	* a consecutive offset. Optimize this special case.
739	*/
740	if (bio->bi_vcnt > 0) {
741	struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
742
743	if (page == prev->bv_page &&
744	offset == prev->bv_offset + prev->bv_len) {
745	prev->bv_len += len;
746	bio->bi_iter.bi_size += len;
747	goto done;
748	}
749
750	/*
751	* If the queue doesn't support SG gaps and adding this
752	* offset would create a gap, disallow it.
753	*/
754	if (bvec_gap_to_prev(q, prev, offset))
755	return 0;
756	}
757
758	if (bio->bi_vcnt >= bio->bi_max_vecs)
759	return 0;
760
761	/*
762	* setup the new entry, we might clear it again later if we
763	* cannot add the page
764	*/
765	bvec = &bio->bi_io_vec[bio->bi_vcnt];
766	bvec->bv_page = page;
767	bvec->bv_len = len;
768	bvec->bv_offset = offset;
769	bio->bi_vcnt++;
770	bio->bi_phys_segments++;
771	bio->bi_iter.bi_size += len;
772
773	/*
774	* Perform a recount if the number of segments is greater
775	* than queue_max_segments(q).
776	*/
777
778	while (bio->bi_phys_segments > queue_max_segments(q)) {
779
780	if (retried_segments)
781	goto failed;
782
783	retried_segments = 1;
784	blk_recount_segments(q, bio);
785	}
786
787	/* If we may be able to merge these biovecs, force a recount */
788	if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
789	bio_clear_flag(bio, BIO_SEG_VALID);
790
791	done:
792	return len;
793
794	failed:
795	bvec->bv_page = NULL;
796	bvec->bv_len = 0;
797	bvec->bv_offset = 0;
798	bio->bi_vcnt--;
799	bio->bi_iter.bi_size -= len;
800	blk_recount_segments(q, bio);
801	return 0;
802	}
803	EXPORT_SYMBOL(bio_add_pc_page);
804
805	/**
806	* bio_add_page - attempt to add page to bio
807	* @bio: destination bio
808	* @page: page to add
809	* @len: vec entry length
810	* @offset: vec entry offset
811	*
812	* Attempt to add a page to the bio_vec maplist. This will only fail
813	* if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
814	*/
815	int bio_add_page(struct bio *bio, struct page *page,
816	unsigned int len, unsigned int offset)
817	{
818	struct bio_vec *bv;
819
820	/*
821	* cloned bio must not modify vec list
822	*/
823	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
824	return 0;
825
826	/*
827	* For filesystems with a blocksize smaller than the pagesize
828	* we will often be called with the same page as last time and
829	* a consecutive offset. Optimize this special case.
830	*/
831	if (bio->bi_vcnt > 0) {
832	bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
833
834	if (page == bv->bv_page &&
835	offset == bv->bv_offset + bv->bv_len) {
836	bv->bv_len += len;
837	goto done;
838	}
839	}
840
841	if (bio->bi_vcnt >= bio->bi_max_vecs)
842	return 0;
843
844	bv = &bio->bi_io_vec[bio->bi_vcnt];
845	bv->bv_page = page;
846	bv->bv_len = len;
847	bv->bv_offset = offset;
848
849	bio->bi_vcnt++;
850	done:
851	bio->bi_iter.bi_size += len;
852	return len;
853	}
854	EXPORT_SYMBOL(bio_add_page);
855
856	struct submit_bio_ret {
857	struct completion event;
858	int error;
859	};
860
861	static void submit_bio_wait_endio(struct bio *bio)
862	{
863	struct submit_bio_ret *ret = bio->bi_private;
864
865	ret->error = bio->bi_error;
866	complete(&ret->event);
867	}
868
869	/**
870	* submit_bio_wait - submit a bio, and wait until it completes
871	* @bio: The &struct bio which describes the I/O
872	*
873	* Simple wrapper around submit_bio(). Returns 0 on success, or the error from
874	* bio_endio() on failure.
875	*/
876	int submit_bio_wait(struct bio *bio)
877	{
878	struct submit_bio_ret ret;
879
880	init_completion(&ret.event);
881	bio->bi_private = &ret;
882	bio->bi_end_io = submit_bio_wait_endio;
883	bio->bi_opf |= REQ_SYNC;
884	submit_bio(bio);
885	wait_for_completion_io(&ret.event);
886
887	return ret.error;
888	}
889	EXPORT_SYMBOL(submit_bio_wait);
890
891	/**
892	* bio_advance - increment/complete a bio by some number of bytes
893	* @bio: bio to advance
894	* @bytes: number of bytes to complete
895	*
896	* This updates bi_sector, bi_size and bi_idx; if the number of bytes to
897	* complete doesn't align with a bvec boundary, then bv_len and bv_offset will
898	* be updated on the last bvec as well.
899	*
900	* @bio will then represent the remaining, uncompleted portion of the io.
901	*/
902	void bio_advance(struct bio *bio, unsigned bytes)
903	{
904	if (bio_integrity(bio))
905	bio_integrity_advance(bio, bytes);
906
907	bio_advance_iter(bio, &bio->bi_iter, bytes);
908	}
909	EXPORT_SYMBOL(bio_advance);
910
911	/**
912	* bio_alloc_pages - allocates a single page for each bvec in a bio
913	* @bio: bio to allocate pages for
914	* @gfp_mask: flags for allocation
915	*
916	* Allocates pages up to @bio->bi_vcnt.
917	*
918	* Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
919	* freed.
920	*/
921	int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
922	{
923	int i;
924	struct bio_vec *bv;
925
926	bio_for_each_segment_all(bv, bio, i) {
927	bv->bv_page = alloc_page(gfp_mask);
928	if (!bv->bv_page) {
929	while (--bv >= bio->bi_io_vec)
930	__free_page(bv->bv_page);
931	return -ENOMEM;
932	}
933	}
934
935	return 0;
936	}
937	EXPORT_SYMBOL(bio_alloc_pages);
938
939	/**
940	* bio_copy_data - copy contents of data buffers from one chain of bios to
941	* another
942	* @src: source bio list
943	* @dst: destination bio list
944	*
945	* If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
946	* @src and @dst as linked lists of bios.
947	*
948	* Stops when it reaches the end of either @src or @dst - that is, copies
949	* min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
950	*/
951	void bio_copy_data(struct bio *dst, struct bio *src)
952	{
953	struct bvec_iter src_iter, dst_iter;
954	struct bio_vec src_bv, dst_bv;
955	void *src_p, *dst_p;
956	unsigned bytes;
957
958	src_iter = src->bi_iter;
959	dst_iter = dst->bi_iter;
960
961	while (1) {
962	if (!src_iter.bi_size) {
963	src = src->bi_next;
964	if (!src)
965	break;
966
967	src_iter = src->bi_iter;
968	}
969
970	if (!dst_iter.bi_size) {
971	dst = dst->bi_next;
972	if (!dst)
973	break;
974
975	dst_iter = dst->bi_iter;
976	}
977
978	src_bv = bio_iter_iovec(src, src_iter);
979	dst_bv = bio_iter_iovec(dst, dst_iter);
980
981	bytes = min(src_bv.bv_len, dst_bv.bv_len);
982
983	src_p = kmap_atomic(src_bv.bv_page);
984	dst_p = kmap_atomic(dst_bv.bv_page);
985
986	memcpy(dst_p + dst_bv.bv_offset,
987	src_p + src_bv.bv_offset,
988	bytes);
989
990	kunmap_atomic(dst_p);
991	kunmap_atomic(src_p);
992
993	bio_advance_iter(src, &src_iter, bytes);
994	bio_advance_iter(dst, &dst_iter, bytes);
995	}
996	}
997	EXPORT_SYMBOL(bio_copy_data);
998
999	struct bio_map_data {
1000	int is_our_pages;
1001	struct iov_iter iter;
1002	struct iovec iov[];
1003	};
1004
1005	static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
1006	gfp_t gfp_mask)
1007	{
1008	if (iov_count > UIO_MAXIOV)
1009	return NULL;
1010
1011	return kmalloc(sizeof(struct bio_map_data) +
1012	sizeof(struct iovec) * iov_count, gfp_mask);
1013	}
1014
1015	/**
1016	* bio_copy_from_iter - copy all pages from iov_iter to bio
1017	* @bio: The &struct bio which describes the I/O as destination
1018	* @iter: iov_iter as source
1019	*
1020	* Copy all pages from iov_iter to bio.
1021	* Returns 0 on success, or error on failure.
1022	*/
1023	static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter)
1024	{
1025	int i;
1026	struct bio_vec *bvec;
1027
1028	bio_for_each_segment_all(bvec, bio, i) {
1029	ssize_t ret;
1030
1031	ret = copy_page_from_iter(bvec->bv_page,
1032	bvec->bv_offset,
1033	bvec->bv_len,
1034	&iter);
1035
1036	if (!iov_iter_count(&iter))
1037	break;
1038
1039	if (ret < bvec->bv_len)
1040	return -EFAULT;
1041	}
1042
1043	return 0;
1044	}
1045
1046	/**
1047	* bio_copy_to_iter - copy all pages from bio to iov_iter
1048	* @bio: The &struct bio which describes the I/O as source
1049	* @iter: iov_iter as destination
1050	*
1051	* Copy all pages from bio to iov_iter.
1052	* Returns 0 on success, or error on failure.
1053	*/
1054	static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
1055	{
1056	int i;
1057	struct bio_vec *bvec;
1058
1059	bio_for_each_segment_all(bvec, bio, i) {
1060	ssize_t ret;
1061
1062	ret = copy_page_to_iter(bvec->bv_page,
1063	bvec->bv_offset,
1064	bvec->bv_len,
1065	&iter);
1066
1067	if (!iov_iter_count(&iter))
1068	break;
1069
1070	if (ret < bvec->bv_len)
1071	return -EFAULT;
1072	}
1073
1074	return 0;
1075	}
1076
1077	void bio_free_pages(struct bio *bio)
1078	{
1079	struct bio_vec *bvec;
1080	int i;
1081
1082	bio_for_each_segment_all(bvec, bio, i)
1083	__free_page(bvec->bv_page);
1084	}
1085	EXPORT_SYMBOL(bio_free_pages);
1086
1087	/**
1088	* bio_uncopy_user - finish previously mapped bio
1089	* @bio: bio being terminated
1090	*
1091	* Free pages allocated from bio_copy_user_iov() and write back data
1092	* to user space in case of a read.
1093	*/
1094	int bio_uncopy_user(struct bio *bio)
1095	{
1096	struct bio_map_data *bmd = bio->bi_private;
1097	int ret = 0;
1098
1099	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
1100	/*
1101	* if we're in a workqueue, the request is orphaned, so
1102	* don't copy into a random user address space, just free
1103	* and return -EINTR so user space doesn't expect any data.
1104	*/
1105	if (!current->mm)
1106	ret = -EINTR;
1107	else if (bio_data_dir(bio) == READ)
1108	ret = bio_copy_to_iter(bio, bmd->iter);
1109	if (bmd->is_our_pages)
1110	bio_free_pages(bio);
1111	}
1112	kfree(bmd);
1113	bio_put(bio);
1114	return ret;
1115	}
1116
1117	/**
1118	* bio_copy_user_iov - copy user data to bio
1119	* @q: destination block queue
1120	* @map_data: pointer to the rq_map_data holding pages (if necessary)
1121	* @iter: iovec iterator
1122	* @gfp_mask: memory allocation flags
1123	*
1124	* Prepares and returns a bio for indirect user io, bouncing data
1125	* to/from kernel pages as necessary. Must be paired with
1126	* call bio_uncopy_user() on io completion.
1127	*/
1128	struct bio *bio_copy_user_iov(struct request_queue *q,
1129	struct rq_map_data *map_data,
1130	const struct iov_iter *iter,
1131	gfp_t gfp_mask)
1132	{
1133	struct bio_map_data *bmd;
1134	struct page *page;
1135	struct bio *bio;
1136	int i, ret;
1137	int nr_pages = 0;
1138	unsigned int len = iter->count;
1139	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
1140
1141	for (i = 0; i < iter->nr_segs; i++) {
1142	unsigned long uaddr;
1143	unsigned long end;
1144	unsigned long start;
1145
1146	uaddr = (unsigned long) iter->iov[i].iov_base;
1147	end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1)
1148	>> PAGE_SHIFT;
1149	start = uaddr >> PAGE_SHIFT;
1150
1151	/*
1152	* Overflow, abort
1153	*/
1154	if (end < start)
1155	return ERR_PTR(-EINVAL);
1156
1157	nr_pages += end - start;
1158	}
1159
1160	if (offset)
1161	nr_pages++;
1162
1163	bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask);
1164	if (!bmd)
1165	return ERR_PTR(-ENOMEM);
1166
1167	/*
1168	* We need to do a deep copy of the iov_iter including the iovecs.
1169	* The caller provided iov might point to an on-stack or otherwise
1170	* shortlived one.
1171	*/
1172	bmd->is_our_pages = map_data ? 0 : 1;
1173	memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs);
1174	bmd->iter = *iter;
1175	bmd->iter.iov = bmd->iov;
1176
1177	ret = -ENOMEM;
1178	bio = bio_kmalloc(gfp_mask, nr_pages);
1179	if (!bio)
1180	goto out_bmd;
1181
1182	if (iter->type & WRITE)
1183	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1184
1185	ret = 0;
1186
1187	if (map_data) {
1188	nr_pages = 1 << map_data->page_order;
1189	i = map_data->offset / PAGE_SIZE;
1190	}
1191	while (len) {
1192	unsigned int bytes = PAGE_SIZE;
1193
1194	bytes -= offset;
1195
1196	if (bytes > len)
1197	bytes = len;
1198
1199	if (map_data) {
1200	if (i == map_data->nr_entries * nr_pages) {
1201	ret = -ENOMEM;
1202	break;
1203	}
1204
1205	page = map_data->pages[i / nr_pages];
1206	page += (i % nr_pages);
1207
1208	i++;
1209	} else {
1210	page = alloc_page(q->bounce_gfp | gfp_mask);
1211	if (!page) {
1212	ret = -ENOMEM;
1213	break;
1214	}
1215	}
1216
1217	if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {
1218	if (!map_data)
1219	__free_page(page);
1220	break;
1221	}
1222
1223	len -= bytes;
1224	offset = 0;
1225	}
1226
1227	if (ret)
1228	goto cleanup;
1229
1230	/*
1231	* success
1232	*/
1233	if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) ||
1234	(map_data && map_data->from_user)) {
1235	ret = bio_copy_from_iter(bio, *iter);
1236	if (ret)
1237	goto cleanup;
1238	}
1239
1240	bio->bi_private = bmd;
1241	return bio;
1242	cleanup:
1243	if (!map_data)
1244	bio_free_pages(bio);
1245	bio_put(bio);
1246	out_bmd:
1247	kfree(bmd);
1248	return ERR_PTR(ret);
1249	}
1250
1251	/**
1252	* bio_map_user_iov - map user iovec into bio
1253	* @q: the struct request_queue for the bio
1254	* @iter: iovec iterator
1255	* @gfp_mask: memory allocation flags
1256	*
1257	* Map the user space address into a bio suitable for io to a block
1258	* device. Returns an error pointer in case of error.
1259	*/
1260	struct bio *bio_map_user_iov(struct request_queue *q,
1261	const struct iov_iter *iter,
1262	gfp_t gfp_mask)
1263	{
1264	int j;
1265	int nr_pages = 0;
1266	struct page **pages;
1267	struct bio *bio;
1268	int cur_page = 0;
1269	int ret, offset;
1270	struct iov_iter i;
1271	struct iovec iov;
1272	struct bio_vec *bvec;
1273
1274	iov_for_each(iov, i, *iter) {
1275	unsigned long uaddr = (unsigned long) iov.iov_base;
1276	unsigned long len = iov.iov_len;
1277	unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1278	unsigned long start = uaddr >> PAGE_SHIFT;
1279
1280	/*
1281	* Overflow, abort
1282	*/
1283	if (end < start)
1284	return ERR_PTR(-EINVAL);
1285
1286	nr_pages += end - start;
1287	/*
1288	* buffer must be aligned to at least logical block size for now
1289	*/
1290	if (uaddr & queue_dma_alignment(q))
1291	return ERR_PTR(-EINVAL);
1292	}
1293
1294	if (!nr_pages)
1295	return ERR_PTR(-EINVAL);
1296
1297	bio = bio_kmalloc(gfp_mask, nr_pages);
1298	if (!bio)
1299	return ERR_PTR(-ENOMEM);
1300
1301	ret = -ENOMEM;
1302	pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
1303	if (!pages)
1304	goto out;
1305
1306	iov_for_each(iov, i, *iter) {
1307	unsigned long uaddr = (unsigned long) iov.iov_base;
1308	unsigned long len = iov.iov_len;
1309	unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1310	unsigned long start = uaddr >> PAGE_SHIFT;
1311	const int local_nr_pages = end - start;
1312	const int page_limit = cur_page + local_nr_pages;
1313
1314	ret = get_user_pages_fast(uaddr, local_nr_pages,
1315	(iter->type & WRITE) != WRITE,
1316	&pages[cur_page]);
1317	if (unlikely(ret < local_nr_pages)) {
1318	for (j = cur_page; j < page_limit; j++) {
1319	if (!pages[j])
1320	break;
1321	put_page(pages[j]);
1322	}
1323	ret = -EFAULT;
1324	goto out_unmap;
1325	}
1326
1327	offset = offset_in_page(uaddr);
1328	for (j = cur_page; j < page_limit; j++) {
1329	unsigned int bytes = PAGE_SIZE - offset;
1330	unsigned short prev_bi_vcnt = bio->bi_vcnt;
1331
1332	if (len <= 0)
1333	break;
1334
1335	if (bytes > len)
1336	bytes = len;
1337
1338	/*
1339	* sorry...
1340	*/
1341	if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
1342	bytes)
1343	break;
1344
1345	/*
1346	* check if vector was merged with previous
1347	* drop page reference if needed
1348	*/
1349	if (bio->bi_vcnt == prev_bi_vcnt)
1350	put_page(pages[j]);
1351
1352	len -= bytes;
1353	offset = 0;
1354	}
1355
1356	cur_page = j;
1357	/*
1358	* release the pages we didn't map into the bio, if any
1359	*/
1360	while (j < page_limit)
1361	put_page(pages[j++]);
1362	}
1363
1364	kfree(pages);
1365
1366	/*
1367	* set data direction, and check if mapped pages need bouncing
1368	*/
1369	if (iter->type & WRITE)
1370	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1371
1372	bio_set_flag(bio, BIO_USER_MAPPED);
1373
1374	/*
1375	* subtle -- if __bio_map_user() ended up bouncing a bio,
1376	* it would normally disappear when its bi_end_io is run.
1377	* however, we need it for the unmap, so grab an extra
1378	* reference to it
1379	*/
1380	bio_get(bio);
1381	return bio;
1382
1383	out_unmap:
1384	bio_for_each_segment_all(bvec, bio, j) {
1385	put_page(bvec->bv_page);
1386	}
1387	out:
1388	kfree(pages);
1389	bio_put(bio);
1390	return ERR_PTR(ret);
1391	}
1392
1393	static void __bio_unmap_user(struct bio *bio)
1394	{
1395	struct bio_vec *bvec;
1396	int i;
1397
1398	/*
1399	* make sure we dirty pages we wrote to
1400	*/
1401	bio_for_each_segment_all(bvec, bio, i) {
1402	if (bio_data_dir(bio) == READ)
1403	set_page_dirty_lock(bvec->bv_page);
1404
1405	put_page(bvec->bv_page);
1406	}
1407
1408	bio_put(bio);
1409	}
1410
1411	/**
1412	* bio_unmap_user - unmap a bio
1413	* @bio: the bio being unmapped
1414	*
1415	* Unmap a bio previously mapped by bio_map_user(). Must be called with
1416	* a process context.
1417	*
1418	* bio_unmap_user() may sleep.
1419	*/
1420	void bio_unmap_user(struct bio *bio)
1421	{
1422	__bio_unmap_user(bio);
1423	bio_put(bio);
1424	}
1425
1426	static void bio_map_kern_endio(struct bio *bio)
1427	{
1428	bio_put(bio);
1429	}
1430
1431	/**
1432	* bio_map_kern - map kernel address into bio
1433	* @q: the struct request_queue for the bio
1434	* @data: pointer to buffer to map
1435	* @len: length in bytes
1436	* @gfp_mask: allocation flags for bio allocation
1437	*
1438	* Map the kernel address into a bio suitable for io to a block
1439	* device. Returns an error pointer in case of error.
1440	*/
1441	struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1442	gfp_t gfp_mask)
1443	{
1444	unsigned long kaddr = (unsigned long)data;
1445	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1446	unsigned long start = kaddr >> PAGE_SHIFT;
1447	const int nr_pages = end - start;
1448	int offset, i;
1449	struct bio *bio;
1450
1451	bio = bio_kmalloc(gfp_mask, nr_pages);
1452	if (!bio)
1453	return ERR_PTR(-ENOMEM);
1454
1455	offset = offset_in_page(kaddr);
1456	for (i = 0; i < nr_pages; i++) {
1457	unsigned int bytes = PAGE_SIZE - offset;
1458
1459	if (len <= 0)
1460	break;
1461
1462	if (bytes > len)
1463	bytes = len;
1464
1465	if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
1466	offset) < bytes) {
1467	/* we don't support partial mappings */
1468	bio_put(bio);
1469	return ERR_PTR(-EINVAL);
1470	}
1471
1472	data += bytes;
1473	len -= bytes;
1474	offset = 0;
1475	}
1476
1477	bio->bi_end_io = bio_map_kern_endio;
1478	return bio;
1479	}
1480	EXPORT_SYMBOL(bio_map_kern);
1481
1482	static void bio_copy_kern_endio(struct bio *bio)
1483	{
1484	bio_free_pages(bio);
1485	bio_put(bio);
1486	}
1487
1488	static void bio_copy_kern_endio_read(struct bio *bio)
1489	{
1490	char *p = bio->bi_private;
1491	struct bio_vec *bvec;
1492	int i;
1493
1494	bio_for_each_segment_all(bvec, bio, i) {
1495	memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
1496	p += bvec->bv_len;
1497	}
1498
1499	bio_copy_kern_endio(bio);
1500	}
1501
1502	/**
1503	* bio_copy_kern - copy kernel address into bio
1504	* @q: the struct request_queue for the bio
1505	* @data: pointer to buffer to copy
1506	* @len: length in bytes
1507	* @gfp_mask: allocation flags for bio and page allocation
1508	* @reading: data direction is READ
1509	*
1510	* copy the kernel address into a bio suitable for io to a block
1511	* device. Returns an error pointer in case of error.
1512	*/
1513	struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1514	gfp_t gfp_mask, int reading)
1515	{
1516	unsigned long kaddr = (unsigned long)data;
1517	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1518	unsigned long start = kaddr >> PAGE_SHIFT;
1519	struct bio *bio;
1520	void *p = data;
1521	int nr_pages = 0;
1522
1523	/*
1524	* Overflow, abort
1525	*/
1526	if (end < start)
1527	return ERR_PTR(-EINVAL);
1528
1529	nr_pages = end - start;
1530	bio = bio_kmalloc(gfp_mask, nr_pages);
1531	if (!bio)
1532	return ERR_PTR(-ENOMEM);
1533
1534	while (len) {
1535	struct page *page;
1536	unsigned int bytes = PAGE_SIZE;
1537
1538	if (bytes > len)
1539	bytes = len;
1540
1541	page = alloc_page(q->bounce_gfp | gfp_mask);
1542	if (!page)
1543	goto cleanup;
1544
1545	if (!reading)
1546	memcpy(page_address(page), p, bytes);
1547
1548	if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
1549	break;
1550
1551	len -= bytes;
1552	p += bytes;
1553	}
1554
1555	if (reading) {
1556	bio->bi_end_io = bio_copy_kern_endio_read;
1557	bio->bi_private = data;
1558	} else {
1559	bio->bi_end_io = bio_copy_kern_endio;
1560	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1561	}
1562
1563	return bio;
1564
1565	cleanup:
1566	bio_free_pages(bio);
1567	bio_put(bio);
1568	return ERR_PTR(-ENOMEM);
1569	}
1570
1571	/*
1572	* bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1573	* for performing direct-IO in BIOs.
1574	*
1575	* The problem is that we cannot run set_page_dirty() from interrupt context
1576	* because the required locks are not interrupt-safe. So what we can do is to
1577	* mark the pages dirty _before_ performing IO. And in interrupt context,
1578	* check that the pages are still dirty. If so, fine. If not, redirty them
1579	* in process context.
1580	*
1581	* We special-case compound pages here: normally this means reads into hugetlb
1582	* pages. The logic in here doesn't really work right for compound pages
1583	* because the VM does not uniformly chase down the head page in all cases.
1584	* But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1585	* handle them at all. So we skip compound pages here at an early stage.
1586	*
1587	* Note that this code is very hard to test under normal circumstances because
1588	* direct-io pins the pages with get_user_pages(). This makes
1589	* is_page_cache_freeable return false, and the VM will not clean the pages.
1590	* But other code (eg, flusher threads) could clean the pages if they are mapped
1591	* pagecache.
1592	*
1593	* Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1594	* deferred bio dirtying paths.
1595	*/
1596
1597	/*
1598	* bio_set_pages_dirty() will mark all the bio's pages as dirty.
1599	*/
1600	void bio_set_pages_dirty(struct bio *bio)
1601	{
1602	struct bio_vec *bvec;
1603	int i;
1604
1605	bio_for_each_segment_all(bvec, bio, i) {
1606	struct page *page = bvec->bv_page;
1607
1608	if (page && !PageCompound(page))
1609	set_page_dirty_lock(page);
1610	}
1611	}
1612
1613	static void bio_release_pages(struct bio *bio)
1614	{
1615	struct bio_vec *bvec;
1616	int i;
1617
1618	bio_for_each_segment_all(bvec, bio, i) {
1619	struct page *page = bvec->bv_page;
1620
1621	if (page)
1622	put_page(page);
1623	}
1624	}
1625
1626	/*
1627	* bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1628	* If they are, then fine. If, however, some pages are clean then they must
1629	* have been written out during the direct-IO read. So we take another ref on
1630	* the BIO and the offending pages and re-dirty the pages in process context.
1631	*
1632	* It is expected that bio_check_pages_dirty() will wholly own the BIO from
1633	* here on. It will run one put_page() against each page and will run one
1634	* bio_put() against the BIO.
1635	*/
1636
1637	static void bio_dirty_fn(struct work_struct *work);
1638
1639	static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1640	static DEFINE_SPINLOCK(bio_dirty_lock);
1641	static struct bio *bio_dirty_list;
1642
1643	/*
1644	* This runs in process context
1645	*/
1646	static void bio_dirty_fn(struct work_struct *work)
1647	{
1648	unsigned long flags;
1649	struct bio *bio;
1650
1651	spin_lock_irqsave(&bio_dirty_lock, flags);
1652	bio = bio_dirty_list;
1653	bio_dirty_list = NULL;
1654	spin_unlock_irqrestore(&bio_dirty_lock, flags);
1655
1656	while (bio) {
1657	struct bio *next = bio->bi_private;
1658
1659	bio_set_pages_dirty(bio);
1660	bio_release_pages(bio);
1661	bio_put(bio);
1662	bio = next;
1663	}
1664	}
1665
1666	void bio_check_pages_dirty(struct bio *bio)
1667	{
1668	struct bio_vec *bvec;
1669	int nr_clean_pages = 0;
1670	int i;
1671
1672	bio_for_each_segment_all(bvec, bio, i) {
1673	struct page *page = bvec->bv_page;
1674
1675	if (PageDirty(page) || PageCompound(page)) {
1676	put_page(page);
1677	bvec->bv_page = NULL;
1678	} else {
1679	nr_clean_pages++;
1680	}
1681	}
1682
1683	if (nr_clean_pages) {
1684	unsigned long flags;
1685
1686	spin_lock_irqsave(&bio_dirty_lock, flags);
1687	bio->bi_private = bio_dirty_list;
1688	bio_dirty_list = bio;
1689	spin_unlock_irqrestore(&bio_dirty_lock, flags);
1690	schedule_work(&bio_dirty_work);
1691	} else {
1692	bio_put(bio);
1693	}
1694	}
1695
1696	void generic_start_io_acct(int rw, unsigned long sectors,
1697	struct hd_struct *part)
1698	{
1699	int cpu = part_stat_lock();
1700
1701	part_round_stats(cpu, part);
1702	part_stat_inc(cpu, part, ios[rw]);
1703	part_stat_add(cpu, part, sectors[rw], sectors);
1704	part_inc_in_flight(part, rw);
1705
1706	part_stat_unlock();
1707	}
1708	EXPORT_SYMBOL(generic_start_io_acct);
1709
1710	void generic_end_io_acct(int rw, struct hd_struct *part,
1711	unsigned long start_time)
1712	{
1713	unsigned long duration = jiffies - start_time;
1714	int cpu = part_stat_lock();
1715
1716	part_stat_add(cpu, part, ticks[rw], duration);
1717	part_round_stats(cpu, part);
1718	part_dec_in_flight(part, rw);
1719
1720	part_stat_unlock();
1721	}
1722	EXPORT_SYMBOL(generic_end_io_acct);
1723
1724	#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1725	void bio_flush_dcache_pages(struct bio *bi)
1726	{
1727	struct bio_vec bvec;
1728	struct bvec_iter iter;
1729
1730	bio_for_each_segment(bvec, bi, iter)
1731	flush_dcache_page(bvec.bv_page);
1732	}
1733	EXPORT_SYMBOL(bio_flush_dcache_pages);
1734	#endif
1735
1736	static inline bool bio_remaining_done(struct bio *bio)
1737	{
1738	/*
1739	* If we're not chaining, then ->__bi_remaining is always 1 and
1740	* we always end io on the first invocation.
1741	*/
1742	if (!bio_flagged(bio, BIO_CHAIN))
1743	return true;
1744
1745	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1746
1747	if (atomic_dec_and_test(&bio->__bi_remaining)) {
1748	bio_clear_flag(bio, BIO_CHAIN);
1749	return true;
1750	}
1751
1752	return false;
1753	}
1754
1755	/**
1756	* bio_endio - end I/O on a bio
1757	* @bio: bio
1758	*
1759	* Description:
1760	* bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1761	* way to end I/O on a bio. No one should call bi_end_io() directly on a
1762	* bio unless they own it and thus know that it has an end_io function.
1763	**/
1764	void bio_endio(struct bio *bio)
1765	{
1766	again:
1767	if (!bio_remaining_done(bio))
1768	return;
1769
1770	/*
1771	* Need to have a real endio function for chained bios, otherwise
1772	* various corner cases will break (like stacking block devices that
1773	* save/restore bi_end_io) - however, we want to avoid unbounded
1774	* recursion and blowing the stack. Tail call optimization would
1775	* handle this, but compiling with frame pointers also disables
1776	* gcc's sibling call optimization.
1777	*/
1778	if (bio->bi_end_io == bio_chain_endio) {
1779	bio = __bio_chain_endio(bio);
1780	goto again;
1781	}
1782
1783	if (bio->bi_end_io)
1784	bio->bi_end_io(bio);
1785	}
1786	EXPORT_SYMBOL(bio_endio);
1787
1788	/**
1789	* bio_split - split a bio
1790	* @bio: bio to split
1791	* @sectors: number of sectors to split from the front of @bio
1792	* @gfp: gfp mask
1793	* @bs: bio set to allocate from
1794	*
1795	* Allocates and returns a new bio which represents @sectors from the start of
1796	* @bio, and updates @bio to represent the remaining sectors.
1797	*
1798	* Unless this is a discard request the newly allocated bio will point
1799	* to @bio's bi_io_vec; it is the caller's responsibility to ensure that
1800	* @bio is not freed before the split.
1801	*/
1802	struct bio *bio_split(struct bio *bio, int sectors,
1803	gfp_t gfp, struct bio_set *bs)
1804	{
1805	struct bio *split = NULL;
1806
1807	BUG_ON(sectors <= 0);
1808	BUG_ON(sectors >= bio_sectors(bio));
1809
1810	/*
1811	* Discards need a mutable bio_vec to accommodate the payload
1812	* required by the DSM TRIM and UNMAP commands.
1813	*/
1814	if (bio_op(bio) == REQ_OP_DISCARD || bio_op(bio) == REQ_OP_SECURE_ERASE)
1815	split = bio_clone_bioset(bio, gfp, bs);
1816	else
1817	split = bio_clone_fast(bio, gfp, bs);
1818
1819	if (!split)
1820	return NULL;
1821
1822	split->bi_iter.bi_size = sectors << 9;
1823
1824	if (bio_integrity(split))
1825	bio_integrity_trim(split, 0, sectors);
1826
1827	bio_advance(bio, split->bi_iter.bi_size);
1828
1829	return split;
1830	}
1831	EXPORT_SYMBOL(bio_split);
1832
1833	/**
1834	* bio_trim - trim a bio
1835	* @bio: bio to trim
1836	* @offset: number of sectors to trim from the front of @bio
1837	* @size: size we want to trim @bio to, in sectors
1838	*/
1839	void bio_trim(struct bio *bio, int offset, int size)
1840	{
1841	/* 'bio' is a cloned bio which we need to trim to match
1842	* the given offset and size.
1843	*/
1844
1845	size <<= 9;
1846	if (offset == 0 && size == bio->bi_iter.bi_size)
1847	return;
1848
1849	bio_clear_flag(bio, BIO_SEG_VALID);
1850
1851	bio_advance(bio, offset << 9);
1852
1853	bio->bi_iter.bi_size = size;
1854	}
1855	EXPORT_SYMBOL_GPL(bio_trim);
1856
1857	/*
1858	* create memory pools for biovec's in a bio_set.
1859	* use the global biovec slabs created for general use.
1860	*/
1861	mempool_t *biovec_create_pool(int pool_entries)
1862	{
1863	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
1864
1865	return mempool_create_slab_pool(pool_entries, bp->slab);
1866	}
1867
1868	void bioset_free(struct bio_set *bs)
1869	{
1870	if (bs->rescue_workqueue)
1871	destroy_workqueue(bs->rescue_workqueue);
1872
1873	if (bs->bio_pool)
1874	mempool_destroy(bs->bio_pool);
1875
1876	if (bs->bvec_pool)
1877	mempool_destroy(bs->bvec_pool);
1878
1879	bioset_integrity_free(bs);
1880	bio_put_slab(bs);
1881
1882	kfree(bs);
1883	}
1884	EXPORT_SYMBOL(bioset_free);
1885
1886	static struct bio_set *__bioset_create(unsigned int pool_size,
1887	unsigned int front_pad,
1888	bool create_bvec_pool)
1889	{
1890	unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1891	struct bio_set *bs;
1892
1893	bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1894	if (!bs)
1895	return NULL;
1896
1897	bs->front_pad = front_pad;
1898
1899	spin_lock_init(&bs->rescue_lock);
1900	bio_list_init(&bs->rescue_list);
1901	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1902
1903	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1904	if (!bs->bio_slab) {
1905	kfree(bs);
1906	return NULL;
1907	}
1908
1909	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1910	if (!bs->bio_pool)
1911	goto bad;
1912
1913	if (create_bvec_pool) {
1914	bs->bvec_pool = biovec_create_pool(pool_size);
1915	if (!bs->bvec_pool)
1916	goto bad;
1917	}
1918
1919	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1920	if (!bs->rescue_workqueue)
1921	goto bad;
1922
1923	return bs;
1924	bad:
1925	bioset_free(bs);
1926	return NULL;
1927	}
1928
1929	/**
1930	* bioset_create - Create a bio_set
1931	* @pool_size: Number of bio and bio_vecs to cache in the mempool
1932	* @front_pad: Number of bytes to allocate in front of the returned bio
1933	*
1934	* Description:
1935	* Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1936	* to ask for a number of bytes to be allocated in front of the bio.
1937	* Front pad allocation is useful for embedding the bio inside
1938	* another structure, to avoid allocating extra data to go with the bio.
1939	* Note that the bio must be embedded at the END of that structure always,
1940	* or things will break badly.
1941	*/
1942	struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1943	{
1944	return __bioset_create(pool_size, front_pad, true);
1945	}
1946	EXPORT_SYMBOL(bioset_create);
1947
1948	/**
1949	* bioset_create_nobvec - Create a bio_set without bio_vec mempool
1950	* @pool_size: Number of bio to cache in the mempool
1951	* @front_pad: Number of bytes to allocate in front of the returned bio
1952	*
1953	* Description:
1954	* Same functionality as bioset_create() except that mempool is not
1955	* created for bio_vecs. Saving some memory for bio_clone_fast() users.
1956	*/
1957	struct bio_set *bioset_create_nobvec(unsigned int pool_size, unsigned int front_pad)
1958	{
1959	return __bioset_create(pool_size, front_pad, false);
1960	}
1961	EXPORT_SYMBOL(bioset_create_nobvec);
1962
1963	#ifdef CONFIG_BLK_CGROUP
1964
1965	/**
1966	* bio_associate_blkcg - associate a bio with the specified blkcg
1967	* @bio: target bio
1968	* @blkcg_css: css of the blkcg to associate
1969	*
1970	* Associate @bio with the blkcg specified by @blkcg_css. Block layer will
1971	* treat @bio as if it were issued by a task which belongs to the blkcg.
1972	*
1973	* This function takes an extra reference of @blkcg_css which will be put
1974	* when @bio is released. The caller must own @bio and is responsible for
1975	* synchronizing calls to this function.
1976	*/
1977	int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css)
1978	{
1979	if (unlikely(bio->bi_css))
1980	return -EBUSY;
1981	css_get(blkcg_css);
1982	bio->bi_css = blkcg_css;
1983	return 0;
1984	}
1985	EXPORT_SYMBOL_GPL(bio_associate_blkcg);
1986
1987	/**
1988	* bio_associate_current - associate a bio with %current
1989	* @bio: target bio
1990	*
1991	* Associate @bio with %current if it hasn't been associated yet. Block
1992	* layer will treat @bio as if it were issued by %current no matter which
1993	* task actually issues it.
1994	*
1995	* This function takes an extra reference of @task's io_context and blkcg
1996	* which will be put when @bio is released. The caller must own @bio,
1997	* ensure %current->io_context exists, and is responsible for synchronizing
1998	* calls to this function.
1999	*/
2000	int bio_associate_current(struct bio *bio)
2001	{
2002	struct io_context *ioc;
2003
2004	if (bio->bi_css)
2005	return -EBUSY;
2006
2007	ioc = current->io_context;
2008	if (!ioc)
2009	return -ENOENT;
2010
2011	get_io_context_active(ioc);
2012	bio->bi_ioc = ioc;
2013	bio->bi_css = task_get_css(current, io_cgrp_id);
2014	return 0;
2015	}
2016	EXPORT_SYMBOL_GPL(bio_associate_current);
2017
2018	/**
2019	* bio_disassociate_task - undo bio_associate_current()
2020	* @bio: target bio
2021	*/
2022	void bio_disassociate_task(struct bio *bio)
2023	{
2024	if (bio->bi_ioc) {
2025	put_io_context(bio->bi_ioc);
2026	bio->bi_ioc = NULL;
2027	}
2028	if (bio->bi_css) {
2029	css_put(bio->bi_css);
2030	bio->bi_css = NULL;
2031	}
2032	}
2033
2034	/**
2035	* bio_clone_blkcg_association - clone blkcg association from src to dst bio
2036	* @dst: destination bio
2037	* @src: source bio
2038	*/
2039	void bio_clone_blkcg_association(struct bio *dst, struct bio *src)
2040	{
2041	if (src->bi_css)
2042	WARN_ON(bio_associate_blkcg(dst, src->bi_css));
2043	}
2044
2045	#endif /* CONFIG_BLK_CGROUP */
2046
2047	static void __init biovec_init_slabs(void)
2048	{
2049	int i;
2050
2051	for (i = 0; i < BVEC_POOL_NR; i++) {
2052	int size;
2053	struct biovec_slab *bvs = bvec_slabs + i;
2054
2055	if (bvs->nr_vecs <= BIO_INLINE_VECS) {
2056	bvs->slab = NULL;
2057	continue;
2058	}
2059
2060	size = bvs->nr_vecs * sizeof(struct bio_vec);
2061	bvs->slab = kmem_cache_create(bvs->name, size, 0,
2062	SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2063	}
2064	}
2065
2066	static int __init init_bio(void)
2067	{
2068	bio_slab_max = 2;
2069	bio_slab_nr = 0;
2070	bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
2071	if (!bio_slabs)
2072	panic("bio: can't allocate bios\n");
2073
2074	bio_integrity_init();
2075	biovec_init_slabs();
2076
2077	fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
2078	if (!fs_bio_set)
2079	panic("bio: can't allocate bios\n");
2080
2081	if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2082	panic("bio: can't create integrity pool\n");
2083
2084	return 0;
2085	}
2086	subsys_initcall(init_bio);
2087