summaryrefslogtreecommitdiff
path: root/fs/read_write.c (plain)
blob: ce77e76a6c89eb9e6f95700ae065f76c5307fb48
1/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
12#include <linux/fsnotify.h>
13#include <linux/security.h>
14#include <linux/export.h>
15#include <linux/syscalls.h>
16#include <linux/pagemap.h>
17#include <linux/splice.h>
18#include <linux/compat.h>
19#include <linux/mount.h>
20#include <linux/fs.h>
21#include "internal.h"
22
23#include <asm/uaccess.h>
24#include <asm/unistd.h>
25
26const struct file_operations generic_ro_fops = {
27 .llseek = generic_file_llseek,
28 .read_iter = generic_file_read_iter,
29 .mmap = generic_file_readonly_mmap,
30 .splice_read = generic_file_splice_read,
31};
32
33EXPORT_SYMBOL(generic_ro_fops);
34
35static inline int unsigned_offsets(struct file *file)
36{
37 return file->f_mode & FMODE_UNSIGNED_OFFSET;
38}
39
40/**
41 * vfs_setpos - update the file offset for lseek
42 * @file: file structure in question
43 * @offset: file offset to seek to
44 * @maxsize: maximum file size
45 *
46 * This is a low-level filesystem helper for updating the file offset to
47 * the value specified by @offset if the given offset is valid and it is
48 * not equal to the current file offset.
49 *
50 * Return the specified offset on success and -EINVAL on invalid offset.
51 */
52loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
53{
54 if (offset < 0 && !unsigned_offsets(file))
55 return -EINVAL;
56 if (offset > maxsize)
57 return -EINVAL;
58
59 if (offset != file->f_pos) {
60 file->f_pos = offset;
61 file->f_version = 0;
62 }
63 return offset;
64}
65EXPORT_SYMBOL(vfs_setpos);
66
67/**
68 * generic_file_llseek_size - generic llseek implementation for regular files
69 * @file: file structure to seek on
70 * @offset: file offset to seek to
71 * @whence: type of seek
72 * @size: max size of this file in file system
73 * @eof: offset used for SEEK_END position
74 *
75 * This is a variant of generic_file_llseek that allows passing in a custom
76 * maximum file size and a custom EOF position, for e.g. hashed directories
77 *
78 * Synchronization:
79 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
80 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
81 * read/writes behave like SEEK_SET against seeks.
82 */
83loff_t
84generic_file_llseek_size(struct file *file, loff_t offset, int whence,
85 loff_t maxsize, loff_t eof)
86{
87 switch (whence) {
88 case SEEK_END:
89 offset += eof;
90 break;
91 case SEEK_CUR:
92 /*
93 * Here we special-case the lseek(fd, 0, SEEK_CUR)
94 * position-querying operation. Avoid rewriting the "same"
95 * f_pos value back to the file because a concurrent read(),
96 * write() or lseek() might have altered it
97 */
98 if (offset == 0)
99 return file->f_pos;
100 /*
101 * f_lock protects against read/modify/write race with other
102 * SEEK_CURs. Note that parallel writes and reads behave
103 * like SEEK_SET.
104 */
105 spin_lock(&file->f_lock);
106 offset = vfs_setpos(file, file->f_pos + offset, maxsize);
107 spin_unlock(&file->f_lock);
108 return offset;
109 case SEEK_DATA:
110 /*
111 * In the generic case the entire file is data, so as long as
112 * offset isn't at the end of the file then the offset is data.
113 */
114 if ((unsigned long long)offset >= eof)
115 return -ENXIO;
116 break;
117 case SEEK_HOLE:
118 /*
119 * There is a virtual hole at the end of the file, so as long as
120 * offset isn't i_size or larger, return i_size.
121 */
122 if ((unsigned long long)offset >= eof)
123 return -ENXIO;
124 offset = eof;
125 break;
126 }
127
128 return vfs_setpos(file, offset, maxsize);
129}
130EXPORT_SYMBOL(generic_file_llseek_size);
131
132/**
133 * generic_file_llseek - generic llseek implementation for regular files
134 * @file: file structure to seek on
135 * @offset: file offset to seek to
136 * @whence: type of seek
137 *
138 * This is a generic implemenation of ->llseek useable for all normal local
139 * filesystems. It just updates the file offset to the value specified by
140 * @offset and @whence.
141 */
142loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
143{
144 struct inode *inode = file->f_mapping->host;
145
146 return generic_file_llseek_size(file, offset, whence,
147 inode->i_sb->s_maxbytes,
148 i_size_read(inode));
149}
150EXPORT_SYMBOL(generic_file_llseek);
151
152/**
153 * fixed_size_llseek - llseek implementation for fixed-sized devices
154 * @file: file structure to seek on
155 * @offset: file offset to seek to
156 * @whence: type of seek
157 * @size: size of the file
158 *
159 */
160loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
161{
162 switch (whence) {
163 case SEEK_SET: case SEEK_CUR: case SEEK_END:
164 return generic_file_llseek_size(file, offset, whence,
165 size, size);
166 default:
167 return -EINVAL;
168 }
169}
170EXPORT_SYMBOL(fixed_size_llseek);
171
172/**
173 * no_seek_end_llseek - llseek implementation for fixed-sized devices
174 * @file: file structure to seek on
175 * @offset: file offset to seek to
176 * @whence: type of seek
177 *
178 */
179loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
180{
181 switch (whence) {
182 case SEEK_SET: case SEEK_CUR:
183 return generic_file_llseek_size(file, offset, whence,
184 OFFSET_MAX, 0);
185 default:
186 return -EINVAL;
187 }
188}
189EXPORT_SYMBOL(no_seek_end_llseek);
190
191/**
192 * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
193 * @file: file structure to seek on
194 * @offset: file offset to seek to
195 * @whence: type of seek
196 * @size: maximal offset allowed
197 *
198 */
199loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
200{
201 switch (whence) {
202 case SEEK_SET: case SEEK_CUR:
203 return generic_file_llseek_size(file, offset, whence,
204 size, 0);
205 default:
206 return -EINVAL;
207 }
208}
209EXPORT_SYMBOL(no_seek_end_llseek_size);
210
211/**
212 * noop_llseek - No Operation Performed llseek implementation
213 * @file: file structure to seek on
214 * @offset: file offset to seek to
215 * @whence: type of seek
216 *
217 * This is an implementation of ->llseek useable for the rare special case when
218 * userspace expects the seek to succeed but the (device) file is actually not
219 * able to perform the seek. In this case you use noop_llseek() instead of
220 * falling back to the default implementation of ->llseek.
221 */
222loff_t noop_llseek(struct file *file, loff_t offset, int whence)
223{
224 return file->f_pos;
225}
226EXPORT_SYMBOL(noop_llseek);
227
228loff_t no_llseek(struct file *file, loff_t offset, int whence)
229{
230 return -ESPIPE;
231}
232EXPORT_SYMBOL(no_llseek);
233
234loff_t default_llseek(struct file *file, loff_t offset, int whence)
235{
236 struct inode *inode = file_inode(file);
237 loff_t retval;
238
239 inode_lock(inode);
240 switch (whence) {
241 case SEEK_END:
242 offset += i_size_read(inode);
243 break;
244 case SEEK_CUR:
245 if (offset == 0) {
246 retval = file->f_pos;
247 goto out;
248 }
249 offset += file->f_pos;
250 break;
251 case SEEK_DATA:
252 /*
253 * In the generic case the entire file is data, so as
254 * long as offset isn't at the end of the file then the
255 * offset is data.
256 */
257 if (offset >= inode->i_size) {
258 retval = -ENXIO;
259 goto out;
260 }
261 break;
262 case SEEK_HOLE:
263 /*
264 * There is a virtual hole at the end of the file, so
265 * as long as offset isn't i_size or larger, return
266 * i_size.
267 */
268 if (offset >= inode->i_size) {
269 retval = -ENXIO;
270 goto out;
271 }
272 offset = inode->i_size;
273 break;
274 }
275 retval = -EINVAL;
276 if (offset >= 0 || unsigned_offsets(file)) {
277 if (offset != file->f_pos) {
278 file->f_pos = offset;
279 file->f_version = 0;
280 }
281 retval = offset;
282 }
283out:
284 inode_unlock(inode);
285 return retval;
286}
287EXPORT_SYMBOL(default_llseek);
288
289loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
290{
291 loff_t (*fn)(struct file *, loff_t, int);
292
293 fn = no_llseek;
294 if (file->f_mode & FMODE_LSEEK) {
295 if (file->f_op->llseek)
296 fn = file->f_op->llseek;
297 }
298 return fn(file, offset, whence);
299}
300EXPORT_SYMBOL(vfs_llseek);
301
302SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
303{
304 off_t retval;
305 struct fd f = fdget_pos(fd);
306 if (!f.file)
307 return -EBADF;
308
309 retval = -EINVAL;
310 if (whence <= SEEK_MAX) {
311 loff_t res = vfs_llseek(f.file, offset, whence);
312 retval = res;
313 if (res != (loff_t)retval)
314 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
315 }
316 fdput_pos(f);
317 return retval;
318}
319
320#ifdef CONFIG_COMPAT
321COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
322{
323 return sys_lseek(fd, offset, whence);
324}
325#endif
326
327#ifdef __ARCH_WANT_SYS_LLSEEK
328SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
329 unsigned long, offset_low, loff_t __user *, result,
330 unsigned int, whence)
331{
332 int retval;
333 struct fd f = fdget_pos(fd);
334 loff_t offset;
335
336 if (!f.file)
337 return -EBADF;
338
339 retval = -EINVAL;
340 if (whence > SEEK_MAX)
341 goto out_putf;
342
343 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
344 whence);
345
346 retval = (int)offset;
347 if (offset >= 0) {
348 retval = -EFAULT;
349 if (!copy_to_user(result, &offset, sizeof(offset)))
350 retval = 0;
351 }
352out_putf:
353 fdput_pos(f);
354 return retval;
355}
356#endif
357
358ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos)
359{
360 struct kiocb kiocb;
361 ssize_t ret;
362
363 if (!file->f_op->read_iter)
364 return -EINVAL;
365
366 init_sync_kiocb(&kiocb, file);
367 kiocb.ki_pos = *ppos;
368
369 iter->type |= READ;
370 ret = file->f_op->read_iter(&kiocb, iter);
371 BUG_ON(ret == -EIOCBQUEUED);
372 if (ret > 0)
373 *ppos = kiocb.ki_pos;
374 return ret;
375}
376EXPORT_SYMBOL(vfs_iter_read);
377
378ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos)
379{
380 struct kiocb kiocb;
381 ssize_t ret;
382
383 if (!file->f_op->write_iter)
384 return -EINVAL;
385
386 init_sync_kiocb(&kiocb, file);
387 kiocb.ki_pos = *ppos;
388
389 iter->type |= WRITE;
390 ret = file->f_op->write_iter(&kiocb, iter);
391 BUG_ON(ret == -EIOCBQUEUED);
392 if (ret > 0) {
393 *ppos = kiocb.ki_pos;
394 fsnotify_modify(file);
395 }
396 return ret;
397}
398EXPORT_SYMBOL(vfs_iter_write);
399
400int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
401{
402 struct inode *inode;
403 loff_t pos;
404 int retval = -EINVAL;
405
406 inode = file_inode(file);
407 if (unlikely((ssize_t) count < 0))
408 return retval;
409 pos = *ppos;
410 if (unlikely(pos < 0)) {
411 if (!unsigned_offsets(file))
412 return retval;
413 if (count >= -pos) /* both values are in 0..LLONG_MAX */
414 return -EOVERFLOW;
415 } else if (unlikely((loff_t) (pos + count) < 0)) {
416 if (!unsigned_offsets(file))
417 return retval;
418 }
419
420 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
421 retval = locks_mandatory_area(inode, file, pos, pos + count - 1,
422 read_write == READ ? F_RDLCK : F_WRLCK);
423 if (retval < 0)
424 return retval;
425 }
426 return security_file_permission(file,
427 read_write == READ ? MAY_READ : MAY_WRITE);
428}
429
430static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
431{
432 struct iovec iov = { .iov_base = buf, .iov_len = len };
433 struct kiocb kiocb;
434 struct iov_iter iter;
435 ssize_t ret;
436
437 init_sync_kiocb(&kiocb, filp);
438 kiocb.ki_pos = *ppos;
439 iov_iter_init(&iter, READ, &iov, 1, len);
440
441 ret = filp->f_op->read_iter(&kiocb, &iter);
442 BUG_ON(ret == -EIOCBQUEUED);
443 *ppos = kiocb.ki_pos;
444 return ret;
445}
446
447ssize_t __vfs_read(struct file *file, char __user *buf, size_t count,
448 loff_t *pos)
449{
450 if (file->f_op->read)
451 return file->f_op->read(file, buf, count, pos);
452 else if (file->f_op->read_iter)
453 return new_sync_read(file, buf, count, pos);
454 else
455 return -EINVAL;
456}
457EXPORT_SYMBOL(__vfs_read);
458
459ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
460{
461 ssize_t ret;
462
463 if (!(file->f_mode & FMODE_READ))
464 return -EBADF;
465 if (!(file->f_mode & FMODE_CAN_READ))
466 return -EINVAL;
467 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
468 return -EFAULT;
469
470 ret = rw_verify_area(READ, file, pos, count);
471 if (!ret) {
472 if (count > MAX_RW_COUNT)
473 count = MAX_RW_COUNT;
474 ret = __vfs_read(file, buf, count, pos);
475 if (ret > 0) {
476 fsnotify_access(file);
477 add_rchar(current, ret);
478 }
479 inc_syscr(current);
480 }
481
482 return ret;
483}
484
485EXPORT_SYMBOL(vfs_read);
486
487static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
488{
489 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
490 struct kiocb kiocb;
491 struct iov_iter iter;
492 ssize_t ret;
493
494 init_sync_kiocb(&kiocb, filp);
495 kiocb.ki_pos = *ppos;
496 iov_iter_init(&iter, WRITE, &iov, 1, len);
497
498 ret = filp->f_op->write_iter(&kiocb, &iter);
499 BUG_ON(ret == -EIOCBQUEUED);
500 if (ret > 0)
501 *ppos = kiocb.ki_pos;
502 return ret;
503}
504
505ssize_t __vfs_write(struct file *file, const char __user *p, size_t count,
506 loff_t *pos)
507{
508 if (file->f_op->write)
509 return file->f_op->write(file, p, count, pos);
510 else if (file->f_op->write_iter)
511 return new_sync_write(file, p, count, pos);
512 else
513 return -EINVAL;
514}
515EXPORT_SYMBOL(__vfs_write);
516
517ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
518{
519 mm_segment_t old_fs;
520 const char __user *p;
521 ssize_t ret;
522
523 if (!(file->f_mode & FMODE_CAN_WRITE))
524 return -EINVAL;
525
526 old_fs = get_fs();
527 set_fs(get_ds());
528 p = (__force const char __user *)buf;
529 if (count > MAX_RW_COUNT)
530 count = MAX_RW_COUNT;
531 ret = __vfs_write(file, p, count, pos);
532 set_fs(old_fs);
533 if (ret > 0) {
534 fsnotify_modify(file);
535 add_wchar(current, ret);
536 }
537 inc_syscw(current);
538 return ret;
539}
540
541EXPORT_SYMBOL(__kernel_write);
542
543ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
544{
545 ssize_t ret;
546
547 if (!(file->f_mode & FMODE_WRITE))
548 return -EBADF;
549 if (!(file->f_mode & FMODE_CAN_WRITE))
550 return -EINVAL;
551 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
552 return -EFAULT;
553
554 ret = rw_verify_area(WRITE, file, pos, count);
555 if (!ret) {
556 if (count > MAX_RW_COUNT)
557 count = MAX_RW_COUNT;
558 file_start_write(file);
559 ret = __vfs_write(file, buf, count, pos);
560 if (ret > 0) {
561 fsnotify_modify(file);
562 add_wchar(current, ret);
563 }
564 inc_syscw(current);
565 file_end_write(file);
566 }
567
568 return ret;
569}
570
571EXPORT_SYMBOL(vfs_write);
572
573static inline loff_t file_pos_read(struct file *file)
574{
575 return file->f_pos;
576}
577
578static inline void file_pos_write(struct file *file, loff_t pos)
579{
580 file->f_pos = pos;
581}
582
583SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
584{
585 struct fd f = fdget_pos(fd);
586 ssize_t ret = -EBADF;
587
588 if (f.file) {
589 loff_t pos = file_pos_read(f.file);
590 ret = vfs_read(f.file, buf, count, &pos);
591 if (ret >= 0)
592 file_pos_write(f.file, pos);
593 fdput_pos(f);
594 }
595 return ret;
596}
597
598SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
599 size_t, count)
600{
601 struct fd f = fdget_pos(fd);
602 ssize_t ret = -EBADF;
603
604 if (f.file) {
605 loff_t pos = file_pos_read(f.file);
606 ret = vfs_write(f.file, buf, count, &pos);
607 if (ret >= 0)
608 file_pos_write(f.file, pos);
609 fdput_pos(f);
610 }
611
612 return ret;
613}
614
615SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
616 size_t, count, loff_t, pos)
617{
618 struct fd f;
619 ssize_t ret = -EBADF;
620
621 if (pos < 0)
622 return -EINVAL;
623
624 f = fdget(fd);
625 if (f.file) {
626 ret = -ESPIPE;
627 if (f.file->f_mode & FMODE_PREAD)
628 ret = vfs_read(f.file, buf, count, &pos);
629 fdput(f);
630 }
631
632 return ret;
633}
634
635SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
636 size_t, count, loff_t, pos)
637{
638 struct fd f;
639 ssize_t ret = -EBADF;
640
641 if (pos < 0)
642 return -EINVAL;
643
644 f = fdget(fd);
645 if (f.file) {
646 ret = -ESPIPE;
647 if (f.file->f_mode & FMODE_PWRITE)
648 ret = vfs_write(f.file, buf, count, &pos);
649 fdput(f);
650 }
651
652 return ret;
653}
654
655/*
656 * Reduce an iovec's length in-place. Return the resulting number of segments
657 */
658unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
659{
660 unsigned long seg = 0;
661 size_t len = 0;
662
663 while (seg < nr_segs) {
664 seg++;
665 if (len + iov->iov_len >= to) {
666 iov->iov_len = to - len;
667 break;
668 }
669 len += iov->iov_len;
670 iov++;
671 }
672 return seg;
673}
674EXPORT_SYMBOL(iov_shorten);
675
676static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
677 loff_t *ppos, int type, int flags)
678{
679 struct kiocb kiocb;
680 ssize_t ret;
681
682 if (flags & ~(RWF_HIPRI | RWF_DSYNC | RWF_SYNC))
683 return -EOPNOTSUPP;
684
685 init_sync_kiocb(&kiocb, filp);
686 if (flags & RWF_HIPRI)
687 kiocb.ki_flags |= IOCB_HIPRI;
688 if (flags & RWF_DSYNC)
689 kiocb.ki_flags |= IOCB_DSYNC;
690 if (flags & RWF_SYNC)
691 kiocb.ki_flags |= (IOCB_DSYNC | IOCB_SYNC);
692 kiocb.ki_pos = *ppos;
693
694 if (type == READ)
695 ret = filp->f_op->read_iter(&kiocb, iter);
696 else
697 ret = filp->f_op->write_iter(&kiocb, iter);
698 BUG_ON(ret == -EIOCBQUEUED);
699 *ppos = kiocb.ki_pos;
700 return ret;
701}
702
703/* Do it by hand, with file-ops */
704static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
705 loff_t *ppos, int type, int flags)
706{
707 ssize_t ret = 0;
708
709 if (flags & ~RWF_HIPRI)
710 return -EOPNOTSUPP;
711
712 while (iov_iter_count(iter)) {
713 struct iovec iovec = iov_iter_iovec(iter);
714 ssize_t nr;
715
716 if (type == READ) {
717 nr = filp->f_op->read(filp, iovec.iov_base,
718 iovec.iov_len, ppos);
719 } else {
720 nr = filp->f_op->write(filp, iovec.iov_base,
721 iovec.iov_len, ppos);
722 }
723
724 if (nr < 0) {
725 if (!ret)
726 ret = nr;
727 break;
728 }
729 ret += nr;
730 if (nr != iovec.iov_len)
731 break;
732 iov_iter_advance(iter, nr);
733 }
734
735 return ret;
736}
737
738/* A write operation does a read from user space and vice versa */
739#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
740
741/**
742 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace
743 * into the kernel and check that it is valid.
744 *
745 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.
746 * @uvector: Pointer to the userspace array.
747 * @nr_segs: Number of elements in userspace array.
748 * @fast_segs: Number of elements in @fast_pointer.
749 * @fast_pointer: Pointer to (usually small on-stack) kernel array.
750 * @ret_pointer: (output parameter) Pointer to a variable that will point to
751 * either @fast_pointer, a newly allocated kernel array, or NULL,
752 * depending on which array was used.
753 *
754 * This function copies an array of &struct iovec of @nr_segs from
755 * userspace into the kernel and checks that each element is valid (e.g.
756 * it does not point to a kernel address or cause overflow by being too
757 * large, etc.).
758 *
759 * As an optimization, the caller may provide a pointer to a small
760 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long
761 * (the size of this array, or 0 if unused, should be given in @fast_segs).
762 *
763 * @ret_pointer will always point to the array that was used, so the
764 * caller must take care not to call kfree() on it e.g. in case the
765 * @fast_pointer array was used and it was allocated on the stack.
766 *
767 * Return: The total number of bytes covered by the iovec array on success
768 * or a negative error code on error.
769 */
770ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
771 unsigned long nr_segs, unsigned long fast_segs,
772 struct iovec *fast_pointer,
773 struct iovec **ret_pointer)
774{
775 unsigned long seg;
776 ssize_t ret;
777 struct iovec *iov = fast_pointer;
778
779 /*
780 * SuS says "The readv() function *may* fail if the iovcnt argument
781 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
782 * traditionally returned zero for zero segments, so...
783 */
784 if (nr_segs == 0) {
785 ret = 0;
786 goto out;
787 }
788
789 /*
790 * First get the "struct iovec" from user memory and
791 * verify all the pointers
792 */
793 if (nr_segs > UIO_MAXIOV) {
794 ret = -EINVAL;
795 goto out;
796 }
797 if (nr_segs > fast_segs) {
798 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
799 if (iov == NULL) {
800 ret = -ENOMEM;
801 goto out;
802 }
803 }
804 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
805 ret = -EFAULT;
806 goto out;
807 }
808
809 /*
810 * According to the Single Unix Specification we should return EINVAL
811 * if an element length is < 0 when cast to ssize_t or if the
812 * total length would overflow the ssize_t return value of the
813 * system call.
814 *
815 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
816 * overflow case.
817 */
818 ret = 0;
819 for (seg = 0; seg < nr_segs; seg++) {
820 void __user *buf = iov[seg].iov_base;
821 ssize_t len = (ssize_t)iov[seg].iov_len;
822
823 /* see if we we're about to use an invalid len or if
824 * it's about to overflow ssize_t */
825 if (len < 0) {
826 ret = -EINVAL;
827 goto out;
828 }
829 if (type >= 0
830 && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
831 ret = -EFAULT;
832 goto out;
833 }
834 if (len > MAX_RW_COUNT - ret) {
835 len = MAX_RW_COUNT - ret;
836 iov[seg].iov_len = len;
837 }
838 ret += len;
839 }
840out:
841 *ret_pointer = iov;
842 return ret;
843}
844
845static ssize_t do_readv_writev(int type, struct file *file,
846 const struct iovec __user * uvector,
847 unsigned long nr_segs, loff_t *pos,
848 int flags)
849{
850 size_t tot_len;
851 struct iovec iovstack[UIO_FASTIOV];
852 struct iovec *iov = iovstack;
853 struct iov_iter iter;
854 ssize_t ret;
855
856 ret = import_iovec(type, uvector, nr_segs,
857 ARRAY_SIZE(iovstack), &iov, &iter);
858 if (ret < 0)
859 return ret;
860
861 tot_len = iov_iter_count(&iter);
862 if (!tot_len)
863 goto out;
864 ret = rw_verify_area(type, file, pos, tot_len);
865 if (ret < 0)
866 goto out;
867
868 if (type != READ)
869 file_start_write(file);
870
871 if ((type == READ && file->f_op->read_iter) ||
872 (type == WRITE && file->f_op->write_iter))
873 ret = do_iter_readv_writev(file, &iter, pos, type, flags);
874 else
875 ret = do_loop_readv_writev(file, &iter, pos, type, flags);
876
877 if (type != READ)
878 file_end_write(file);
879
880out:
881 kfree(iov);
882 if ((ret + (type == READ)) > 0) {
883 if (type == READ)
884 fsnotify_access(file);
885 else
886 fsnotify_modify(file);
887 }
888 return ret;
889}
890
891ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
892 unsigned long vlen, loff_t *pos, int flags)
893{
894 if (!(file->f_mode & FMODE_READ))
895 return -EBADF;
896 if (!(file->f_mode & FMODE_CAN_READ))
897 return -EINVAL;
898
899 return do_readv_writev(READ, file, vec, vlen, pos, flags);
900}
901
902EXPORT_SYMBOL(vfs_readv);
903
904ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
905 unsigned long vlen, loff_t *pos, int flags)
906{
907 if (!(file->f_mode & FMODE_WRITE))
908 return -EBADF;
909 if (!(file->f_mode & FMODE_CAN_WRITE))
910 return -EINVAL;
911
912 return do_readv_writev(WRITE, file, vec, vlen, pos, flags);
913}
914
915EXPORT_SYMBOL(vfs_writev);
916
917static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
918 unsigned long vlen, int flags)
919{
920 struct fd f = fdget_pos(fd);
921 ssize_t ret = -EBADF;
922
923 if (f.file) {
924 loff_t pos = file_pos_read(f.file);
925 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
926 if (ret >= 0)
927 file_pos_write(f.file, pos);
928 fdput_pos(f);
929 }
930
931 if (ret > 0)
932 add_rchar(current, ret);
933 inc_syscr(current);
934 return ret;
935}
936
937static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
938 unsigned long vlen, int flags)
939{
940 struct fd f = fdget_pos(fd);
941 ssize_t ret = -EBADF;
942
943 if (f.file) {
944 loff_t pos = file_pos_read(f.file);
945 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
946 if (ret >= 0)
947 file_pos_write(f.file, pos);
948 fdput_pos(f);
949 }
950
951 if (ret > 0)
952 add_wchar(current, ret);
953 inc_syscw(current);
954 return ret;
955}
956
957static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
958{
959#define HALF_LONG_BITS (BITS_PER_LONG / 2)
960 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
961}
962
963static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
964 unsigned long vlen, loff_t pos, int flags)
965{
966 struct fd f;
967 ssize_t ret = -EBADF;
968
969 if (pos < 0)
970 return -EINVAL;
971
972 f = fdget(fd);
973 if (f.file) {
974 ret = -ESPIPE;
975 if (f.file->f_mode & FMODE_PREAD)
976 ret = vfs_readv(f.file, vec, vlen, &pos, flags);
977 fdput(f);
978 }
979
980 if (ret > 0)
981 add_rchar(current, ret);
982 inc_syscr(current);
983 return ret;
984}
985
986static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
987 unsigned long vlen, loff_t pos, int flags)
988{
989 struct fd f;
990 ssize_t ret = -EBADF;
991
992 if (pos < 0)
993 return -EINVAL;
994
995 f = fdget(fd);
996 if (f.file) {
997 ret = -ESPIPE;
998 if (f.file->f_mode & FMODE_PWRITE)
999 ret = vfs_writev(f.file, vec, vlen, &pos, flags);
1000 fdput(f);
1001 }
1002
1003 if (ret > 0)
1004 add_wchar(current, ret);
1005 inc_syscw(current);
1006 return ret;
1007}
1008
1009SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1010 unsigned long, vlen)
1011{
1012 return do_readv(fd, vec, vlen, 0);
1013}
1014
1015SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1016 unsigned long, vlen)
1017{
1018 return do_writev(fd, vec, vlen, 0);
1019}
1020
1021SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1022 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1023{
1024 loff_t pos = pos_from_hilo(pos_h, pos_l);
1025
1026 return do_preadv(fd, vec, vlen, pos, 0);
1027}
1028
1029SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1030 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1031 int, flags)
1032{
1033 loff_t pos = pos_from_hilo(pos_h, pos_l);
1034
1035 if (pos == -1)
1036 return do_readv(fd, vec, vlen, flags);
1037
1038 return do_preadv(fd, vec, vlen, pos, flags);
1039}
1040
1041SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1042 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1043{
1044 loff_t pos = pos_from_hilo(pos_h, pos_l);
1045
1046 return do_pwritev(fd, vec, vlen, pos, 0);
1047}
1048
1049SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1050 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1051 int, flags)
1052{
1053 loff_t pos = pos_from_hilo(pos_h, pos_l);
1054
1055 if (pos == -1)
1056 return do_writev(fd, vec, vlen, flags);
1057
1058 return do_pwritev(fd, vec, vlen, pos, flags);
1059}
1060
1061#ifdef CONFIG_COMPAT
1062
1063static ssize_t compat_do_readv_writev(int type, struct file *file,
1064 const struct compat_iovec __user *uvector,
1065 unsigned long nr_segs, loff_t *pos,
1066 int flags)
1067{
1068 compat_ssize_t tot_len;
1069 struct iovec iovstack[UIO_FASTIOV];
1070 struct iovec *iov = iovstack;
1071 struct iov_iter iter;
1072 ssize_t ret;
1073
1074 ret = compat_import_iovec(type, uvector, nr_segs,
1075 UIO_FASTIOV, &iov, &iter);
1076 if (ret < 0)
1077 return ret;
1078
1079 tot_len = iov_iter_count(&iter);
1080 if (!tot_len)
1081 goto out;
1082 ret = rw_verify_area(type, file, pos, tot_len);
1083 if (ret < 0)
1084 goto out;
1085
1086 if (type != READ)
1087 file_start_write(file);
1088
1089 if ((type == READ && file->f_op->read_iter) ||
1090 (type == WRITE && file->f_op->write_iter))
1091 ret = do_iter_readv_writev(file, &iter, pos, type, flags);
1092 else
1093 ret = do_loop_readv_writev(file, &iter, pos, type, flags);
1094
1095 if (type != READ)
1096 file_end_write(file);
1097
1098out:
1099 kfree(iov);
1100 if ((ret + (type == READ)) > 0) {
1101 if (type == READ)
1102 fsnotify_access(file);
1103 else
1104 fsnotify_modify(file);
1105 }
1106 return ret;
1107}
1108
1109static size_t compat_readv(struct file *file,
1110 const struct compat_iovec __user *vec,
1111 unsigned long vlen, loff_t *pos, int flags)
1112{
1113 ssize_t ret = -EBADF;
1114
1115 if (!(file->f_mode & FMODE_READ))
1116 goto out;
1117
1118 ret = -EINVAL;
1119 if (!(file->f_mode & FMODE_CAN_READ))
1120 goto out;
1121
1122 ret = compat_do_readv_writev(READ, file, vec, vlen, pos, flags);
1123
1124out:
1125 if (ret > 0)
1126 add_rchar(current, ret);
1127 inc_syscr(current);
1128 return ret;
1129}
1130
1131static size_t do_compat_readv(compat_ulong_t fd,
1132 const struct compat_iovec __user *vec,
1133 compat_ulong_t vlen, int flags)
1134{
1135 struct fd f = fdget_pos(fd);
1136 ssize_t ret;
1137 loff_t pos;
1138
1139 if (!f.file)
1140 return -EBADF;
1141 pos = f.file->f_pos;
1142 ret = compat_readv(f.file, vec, vlen, &pos, flags);
1143 if (ret >= 0)
1144 f.file->f_pos = pos;
1145 fdput_pos(f);
1146 return ret;
1147
1148}
1149
1150COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
1151 const struct compat_iovec __user *,vec,
1152 compat_ulong_t, vlen)
1153{
1154 return do_compat_readv(fd, vec, vlen, 0);
1155}
1156
1157static long do_compat_preadv64(unsigned long fd,
1158 const struct compat_iovec __user *vec,
1159 unsigned long vlen, loff_t pos, int flags)
1160{
1161 struct fd f;
1162 ssize_t ret;
1163
1164 if (pos < 0)
1165 return -EINVAL;
1166 f = fdget(fd);
1167 if (!f.file)
1168 return -EBADF;
1169 ret = -ESPIPE;
1170 if (f.file->f_mode & FMODE_PREAD)
1171 ret = compat_readv(f.file, vec, vlen, &pos, flags);
1172 fdput(f);
1173 return ret;
1174}
1175
1176#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1177COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1178 const struct compat_iovec __user *,vec,
1179 unsigned long, vlen, loff_t, pos)
1180{
1181 return do_compat_preadv64(fd, vec, vlen, pos, 0);
1182}
1183#endif
1184
1185COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1186 const struct compat_iovec __user *,vec,
1187 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1188{
1189 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1190
1191 return do_compat_preadv64(fd, vec, vlen, pos, 0);
1192}
1193
1194#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1195COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1196 const struct compat_iovec __user *,vec,
1197 unsigned long, vlen, loff_t, pos, int, flags)
1198{
1199 if (pos == -1)
1200 return do_compat_readv(fd, vec, vlen, flags);
1201
1202 return do_compat_preadv64(fd, vec, vlen, pos, flags);
1203}
1204#endif
1205
1206COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1207 const struct compat_iovec __user *,vec,
1208 compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1209 int, flags)
1210{
1211 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1212
1213 if (pos == -1)
1214 return do_compat_readv(fd, vec, vlen, flags);
1215
1216 return do_compat_preadv64(fd, vec, vlen, pos, flags);
1217}
1218
1219static size_t compat_writev(struct file *file,
1220 const struct compat_iovec __user *vec,
1221 unsigned long vlen, loff_t *pos, int flags)
1222{
1223 ssize_t ret = -EBADF;
1224
1225 if (!(file->f_mode & FMODE_WRITE))
1226 goto out;
1227
1228 ret = -EINVAL;
1229 if (!(file->f_mode & FMODE_CAN_WRITE))
1230 goto out;
1231
1232 ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos, flags);
1233
1234out:
1235 if (ret > 0)
1236 add_wchar(current, ret);
1237 inc_syscw(current);
1238 return ret;
1239}
1240
1241static size_t do_compat_writev(compat_ulong_t fd,
1242 const struct compat_iovec __user* vec,
1243 compat_ulong_t vlen, int flags)
1244{
1245 struct fd f = fdget_pos(fd);
1246 ssize_t ret;
1247 loff_t pos;
1248
1249 if (!f.file)
1250 return -EBADF;
1251 pos = f.file->f_pos;
1252 ret = compat_writev(f.file, vec, vlen, &pos, flags);
1253 if (ret >= 0)
1254 f.file->f_pos = pos;
1255 fdput_pos(f);
1256 return ret;
1257}
1258
1259COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
1260 const struct compat_iovec __user *, vec,
1261 compat_ulong_t, vlen)
1262{
1263 return do_compat_writev(fd, vec, vlen, 0);
1264}
1265
1266static long do_compat_pwritev64(unsigned long fd,
1267 const struct compat_iovec __user *vec,
1268 unsigned long vlen, loff_t pos, int flags)
1269{
1270 struct fd f;
1271 ssize_t ret;
1272
1273 if (pos < 0)
1274 return -EINVAL;
1275 f = fdget(fd);
1276 if (!f.file)
1277 return -EBADF;
1278 ret = -ESPIPE;
1279 if (f.file->f_mode & FMODE_PWRITE)
1280 ret = compat_writev(f.file, vec, vlen, &pos, flags);
1281 fdput(f);
1282 return ret;
1283}
1284
1285#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1286COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1287 const struct compat_iovec __user *,vec,
1288 unsigned long, vlen, loff_t, pos)
1289{
1290 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
1291}
1292#endif
1293
1294COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1295 const struct compat_iovec __user *,vec,
1296 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1297{
1298 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1299
1300 return do_compat_pwritev64(fd, vec, vlen, pos, 0);
1301}
1302
1303#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1304COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1305 const struct compat_iovec __user *,vec,
1306 unsigned long, vlen, loff_t, pos, int, flags)
1307{
1308 if (pos == -1)
1309 return do_compat_writev(fd, vec, vlen, flags);
1310
1311 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1312}
1313#endif
1314
1315COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1316 const struct compat_iovec __user *,vec,
1317 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, int, flags)
1318{
1319 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1320
1321 if (pos == -1)
1322 return do_compat_writev(fd, vec, vlen, flags);
1323
1324 return do_compat_pwritev64(fd, vec, vlen, pos, flags);
1325}
1326
1327#endif
1328
1329static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1330 size_t count, loff_t max)
1331{
1332 struct fd in, out;
1333 struct inode *in_inode, *out_inode;
1334 loff_t pos;
1335 loff_t out_pos;
1336 ssize_t retval;
1337 int fl;
1338
1339 /*
1340 * Get input file, and verify that it is ok..
1341 */
1342 retval = -EBADF;
1343 in = fdget(in_fd);
1344 if (!in.file)
1345 goto out;
1346 if (!(in.file->f_mode & FMODE_READ))
1347 goto fput_in;
1348 retval = -ESPIPE;
1349 if (!ppos) {
1350 pos = in.file->f_pos;
1351 } else {
1352 pos = *ppos;
1353 if (!(in.file->f_mode & FMODE_PREAD))
1354 goto fput_in;
1355 }
1356 retval = rw_verify_area(READ, in.file, &pos, count);
1357 if (retval < 0)
1358 goto fput_in;
1359 if (count > MAX_RW_COUNT)
1360 count = MAX_RW_COUNT;
1361
1362 /*
1363 * Get output file, and verify that it is ok..
1364 */
1365 retval = -EBADF;
1366 out = fdget(out_fd);
1367 if (!out.file)
1368 goto fput_in;
1369 if (!(out.file->f_mode & FMODE_WRITE))
1370 goto fput_out;
1371 retval = -EINVAL;
1372 in_inode = file_inode(in.file);
1373 out_inode = file_inode(out.file);
1374 out_pos = out.file->f_pos;
1375 retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1376 if (retval < 0)
1377 goto fput_out;
1378
1379 if (!max)
1380 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1381
1382 if (unlikely(pos + count > max)) {
1383 retval = -EOVERFLOW;
1384 if (pos >= max)
1385 goto fput_out;
1386 count = max - pos;
1387 }
1388
1389 fl = 0;
1390#if 0
1391 /*
1392 * We need to debate whether we can enable this or not. The
1393 * man page documents EAGAIN return for the output at least,
1394 * and the application is arguably buggy if it doesn't expect
1395 * EAGAIN on a non-blocking file descriptor.
1396 */
1397 if (in.file->f_flags & O_NONBLOCK)
1398 fl = SPLICE_F_NONBLOCK;
1399#endif
1400 file_start_write(out.file);
1401 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
1402 file_end_write(out.file);
1403
1404 if (retval > 0) {
1405 add_rchar(current, retval);
1406 add_wchar(current, retval);
1407 fsnotify_access(in.file);
1408 fsnotify_modify(out.file);
1409 out.file->f_pos = out_pos;
1410 if (ppos)
1411 *ppos = pos;
1412 else
1413 in.file->f_pos = pos;
1414 }
1415
1416 inc_syscr(current);
1417 inc_syscw(current);
1418 if (pos > max)
1419 retval = -EOVERFLOW;
1420
1421fput_out:
1422 fdput(out);
1423fput_in:
1424 fdput(in);
1425out:
1426 return retval;
1427}
1428
1429SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1430{
1431 loff_t pos;
1432 off_t off;
1433 ssize_t ret;
1434
1435 if (offset) {
1436 if (unlikely(get_user(off, offset)))
1437 return -EFAULT;
1438 pos = off;
1439 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1440 if (unlikely(put_user(pos, offset)))
1441 return -EFAULT;
1442 return ret;
1443 }
1444
1445 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1446}
1447
1448SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1449{
1450 loff_t pos;
1451 ssize_t ret;
1452
1453 if (offset) {
1454 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1455 return -EFAULT;
1456 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1457 if (unlikely(put_user(pos, offset)))
1458 return -EFAULT;
1459 return ret;
1460 }
1461
1462 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1463}
1464
1465#ifdef CONFIG_COMPAT
1466COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1467 compat_off_t __user *, offset, compat_size_t, count)
1468{
1469 loff_t pos;
1470 off_t off;
1471 ssize_t ret;
1472
1473 if (offset) {
1474 if (unlikely(get_user(off, offset)))
1475 return -EFAULT;
1476 pos = off;
1477 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1478 if (unlikely(put_user(pos, offset)))
1479 return -EFAULT;
1480 return ret;
1481 }
1482
1483 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1484}
1485
1486COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1487 compat_loff_t __user *, offset, compat_size_t, count)
1488{
1489 loff_t pos;
1490 ssize_t ret;
1491
1492 if (offset) {
1493 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1494 return -EFAULT;
1495 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1496 if (unlikely(put_user(pos, offset)))
1497 return -EFAULT;
1498 return ret;
1499 }
1500
1501 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1502}
1503#endif
1504
1505/*
1506 * copy_file_range() differs from regular file read and write in that it
1507 * specifically allows return partial success. When it does so is up to
1508 * the copy_file_range method.
1509 */
1510ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1511 struct file *file_out, loff_t pos_out,
1512 size_t len, unsigned int flags)
1513{
1514 struct inode *inode_in = file_inode(file_in);
1515 struct inode *inode_out = file_inode(file_out);
1516 ssize_t ret;
1517
1518 if (flags != 0)
1519 return -EINVAL;
1520
1521 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1522 return -EISDIR;
1523 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1524 return -EINVAL;
1525
1526 ret = rw_verify_area(READ, file_in, &pos_in, len);
1527 if (unlikely(ret))
1528 return ret;
1529
1530 ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1531 if (unlikely(ret))
1532 return ret;
1533
1534 if (!(file_in->f_mode & FMODE_READ) ||
1535 !(file_out->f_mode & FMODE_WRITE) ||
1536 (file_out->f_flags & O_APPEND))
1537 return -EBADF;
1538
1539 /* this could be relaxed once a method supports cross-fs copies */
1540 if (inode_in->i_sb != inode_out->i_sb)
1541 return -EXDEV;
1542
1543 if (len == 0)
1544 return 0;
1545
1546 ret = mnt_want_write_file(file_out);
1547 if (ret)
1548 return ret;
1549
1550 ret = -EOPNOTSUPP;
1551 if (file_out->f_op->copy_file_range)
1552 ret = file_out->f_op->copy_file_range(file_in, pos_in, file_out,
1553 pos_out, len, flags);
1554 if (ret == -EOPNOTSUPP)
1555 ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1556 len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1557
1558 if (ret > 0) {
1559 fsnotify_access(file_in);
1560 add_rchar(current, ret);
1561 fsnotify_modify(file_out);
1562 add_wchar(current, ret);
1563 }
1564 inc_syscr(current);
1565 inc_syscw(current);
1566
1567 mnt_drop_write_file(file_out);
1568
1569 return ret;
1570}
1571EXPORT_SYMBOL(vfs_copy_file_range);
1572
1573SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1574 int, fd_out, loff_t __user *, off_out,
1575 size_t, len, unsigned int, flags)
1576{
1577 loff_t pos_in;
1578 loff_t pos_out;
1579 struct fd f_in;
1580 struct fd f_out;
1581 ssize_t ret = -EBADF;
1582
1583 f_in = fdget(fd_in);
1584 if (!f_in.file)
1585 goto out2;
1586
1587 f_out = fdget(fd_out);
1588 if (!f_out.file)
1589 goto out1;
1590
1591 ret = -EFAULT;
1592 if (off_in) {
1593 if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1594 goto out;
1595 } else {
1596 pos_in = f_in.file->f_pos;
1597 }
1598
1599 if (off_out) {
1600 if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1601 goto out;
1602 } else {
1603 pos_out = f_out.file->f_pos;
1604 }
1605
1606 ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
1607 flags);
1608 if (ret > 0) {
1609 pos_in += ret;
1610 pos_out += ret;
1611
1612 if (off_in) {
1613 if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1614 ret = -EFAULT;
1615 } else {
1616 f_in.file->f_pos = pos_in;
1617 }
1618
1619 if (off_out) {
1620 if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1621 ret = -EFAULT;
1622 } else {
1623 f_out.file->f_pos = pos_out;
1624 }
1625 }
1626
1627out:
1628 fdput(f_out);
1629out1:
1630 fdput(f_in);
1631out2:
1632 return ret;
1633}
1634
1635static int clone_verify_area(struct file *file, loff_t pos, u64 len, bool write)
1636{
1637 struct inode *inode = file_inode(file);
1638
1639 if (unlikely(pos < 0))
1640 return -EINVAL;
1641
1642 if (unlikely((loff_t) (pos + len) < 0))
1643 return -EINVAL;
1644
1645 if (unlikely(inode->i_flctx && mandatory_lock(inode))) {
1646 loff_t end = len ? pos + len - 1 : OFFSET_MAX;
1647 int retval;
1648
1649 retval = locks_mandatory_area(inode, file, pos, end,
1650 write ? F_WRLCK : F_RDLCK);
1651 if (retval < 0)
1652 return retval;
1653 }
1654
1655 return security_file_permission(file, write ? MAY_WRITE : MAY_READ);
1656}
1657
1658int vfs_clone_file_range(struct file *file_in, loff_t pos_in,
1659 struct file *file_out, loff_t pos_out, u64 len)
1660{
1661 struct inode *inode_in = file_inode(file_in);
1662 struct inode *inode_out = file_inode(file_out);
1663 int ret;
1664
1665 if (inode_in->i_sb != inode_out->i_sb ||
1666 file_in->f_path.mnt != file_out->f_path.mnt)
1667 return -EXDEV;
1668
1669 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1670 return -EISDIR;
1671 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1672 return -EINVAL;
1673
1674 if (!(file_in->f_mode & FMODE_READ) ||
1675 !(file_out->f_mode & FMODE_WRITE) ||
1676 (file_out->f_flags & O_APPEND))
1677 return -EBADF;
1678
1679 if (!file_in->f_op->clone_file_range)
1680 return -EOPNOTSUPP;
1681
1682 ret = clone_verify_area(file_in, pos_in, len, false);
1683 if (ret)
1684 return ret;
1685
1686 ret = clone_verify_area(file_out, pos_out, len, true);
1687 if (ret)
1688 return ret;
1689
1690 if (pos_in + len > i_size_read(inode_in))
1691 return -EINVAL;
1692
1693 ret = mnt_want_write_file(file_out);
1694 if (ret)
1695 return ret;
1696
1697 ret = file_in->f_op->clone_file_range(file_in, pos_in,
1698 file_out, pos_out, len);
1699 if (!ret) {
1700 fsnotify_access(file_in);
1701 fsnotify_modify(file_out);
1702 }
1703
1704 mnt_drop_write_file(file_out);
1705 return ret;
1706}
1707EXPORT_SYMBOL(vfs_clone_file_range);
1708
1709int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
1710{
1711 struct file_dedupe_range_info *info;
1712 struct inode *src = file_inode(file);
1713 u64 off;
1714 u64 len;
1715 int i;
1716 int ret;
1717 bool is_admin = capable(CAP_SYS_ADMIN);
1718 u16 count = same->dest_count;
1719 struct file *dst_file;
1720 loff_t dst_off;
1721 ssize_t deduped;
1722
1723 if (!(file->f_mode & FMODE_READ))
1724 return -EINVAL;
1725
1726 if (same->reserved1 || same->reserved2)
1727 return -EINVAL;
1728
1729 off = same->src_offset;
1730 len = same->src_length;
1731
1732 ret = -EISDIR;
1733 if (S_ISDIR(src->i_mode))
1734 goto out;
1735
1736 ret = -EINVAL;
1737 if (!S_ISREG(src->i_mode))
1738 goto out;
1739
1740 ret = clone_verify_area(file, off, len, false);
1741 if (ret < 0)
1742 goto out;
1743 ret = 0;
1744
1745 /* pre-format output fields to sane values */
1746 for (i = 0; i < count; i++) {
1747 same->info[i].bytes_deduped = 0ULL;
1748 same->info[i].status = FILE_DEDUPE_RANGE_SAME;
1749 }
1750
1751 for (i = 0, info = same->info; i < count; i++, info++) {
1752 struct inode *dst;
1753 struct fd dst_fd = fdget(info->dest_fd);
1754
1755 dst_file = dst_fd.file;
1756 if (!dst_file) {
1757 info->status = -EBADF;
1758 goto next_loop;
1759 }
1760 dst = file_inode(dst_file);
1761
1762 ret = mnt_want_write_file(dst_file);
1763 if (ret) {
1764 info->status = ret;
1765 goto next_loop;
1766 }
1767
1768 dst_off = info->dest_offset;
1769 ret = clone_verify_area(dst_file, dst_off, len, true);
1770 if (ret < 0) {
1771 info->status = ret;
1772 goto next_file;
1773 }
1774 ret = 0;
1775
1776 if (info->reserved) {
1777 info->status = -EINVAL;
1778 } else if (!(is_admin || (dst_file->f_mode & FMODE_WRITE))) {
1779 info->status = -EINVAL;
1780 } else if (file->f_path.mnt != dst_file->f_path.mnt) {
1781 info->status = -EXDEV;
1782 } else if (S_ISDIR(dst->i_mode)) {
1783 info->status = -EISDIR;
1784 } else if (dst_file->f_op->dedupe_file_range == NULL) {
1785 info->status = -EINVAL;
1786 } else {
1787 deduped = dst_file->f_op->dedupe_file_range(file, off,
1788 len, dst_file,
1789 info->dest_offset);
1790 if (deduped == -EBADE)
1791 info->status = FILE_DEDUPE_RANGE_DIFFERS;
1792 else if (deduped < 0)
1793 info->status = deduped;
1794 else
1795 info->bytes_deduped += deduped;
1796 }
1797
1798next_file:
1799 mnt_drop_write_file(dst_file);
1800next_loop:
1801 fdput(dst_fd);
1802
1803 if (fatal_signal_pending(current))
1804 goto out;
1805 }
1806
1807out:
1808 return ret;
1809}
1810EXPORT_SYMBOL(vfs_dedupe_file_range);
1811
generated by cgit at 2024-05-18 09:02:38 (GMT)