blob: d73eed0443f698a4b97f443f9e909dc765b0e97c
1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> |
3 | #include <linux/err.h> |
4 | #include <linux/mm.h> |
5 | #include <linux/slab.h> |
6 | #include <linux/vmalloc.h> |
7 | #include <linux/pagemap.h> |
8 | #include <linux/sched.h> |
9 | |
10 | /** |
11 | * get_vaddr_frames() - map virtual addresses to pfns |
12 | * @start: starting user address |
13 | * @nr_frames: number of pages / pfns from start to map |
14 | * @gup_flags: flags modifying lookup behaviour |
15 | * @vec: structure which receives pages / pfns of the addresses mapped. |
16 | * It should have space for at least nr_frames entries. |
17 | * |
18 | * This function maps virtual addresses from @start and fills @vec structure |
19 | * with page frame numbers or page pointers to corresponding pages (choice |
20 | * depends on the type of the vma underlying the virtual address). If @start |
21 | * belongs to a normal vma, the function grabs reference to each of the pages |
22 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't |
23 | * touch page structures and the caller must make sure pfns aren't reused for |
24 | * anything else while he is using them. |
25 | * |
26 | * The function returns number of pages mapped which may be less than |
27 | * @nr_frames. In particular we stop mapping if there are more vmas of |
28 | * different type underlying the specified range of virtual addresses. |
29 | * When the function isn't able to map a single page, it returns error. |
30 | * |
31 | * This function takes care of grabbing mmap_sem as necessary. |
32 | */ |
33 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, |
34 | unsigned int gup_flags, struct frame_vector *vec) |
35 | { |
36 | struct mm_struct *mm = current->mm; |
37 | struct vm_area_struct *vma; |
38 | int ret = 0; |
39 | int err; |
40 | int locked; |
41 | |
42 | if (nr_frames == 0) |
43 | return 0; |
44 | |
45 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) |
46 | nr_frames = vec->nr_allocated; |
47 | |
48 | down_read(&mm->mmap_sem); |
49 | locked = 1; |
50 | vma = find_vma_intersection(mm, start, start + 1); |
51 | if (!vma) { |
52 | ret = -EFAULT; |
53 | goto out; |
54 | } |
55 | |
56 | /* |
57 | * While get_vaddr_frames() could be used for transient (kernel |
58 | * controlled lifetime) pinning of memory pages all current |
59 | * users establish long term (userspace controlled lifetime) |
60 | * page pinning. Treat get_vaddr_frames() like |
61 | * get_user_pages_longterm() and disallow it for filesystem-dax |
62 | * mappings. |
63 | */ |
64 | if (vma_is_fsdax(vma)) { |
65 | ret = -EOPNOTSUPP; |
66 | goto out; |
67 | } |
68 | |
69 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { |
70 | vec->got_ref = true; |
71 | vec->is_pfns = false; |
72 | ret = get_user_pages_locked(start, nr_frames, |
73 | gup_flags, (struct page **)(vec->ptrs), &locked); |
74 | goto out; |
75 | } |
76 | |
77 | vec->got_ref = false; |
78 | vec->is_pfns = true; |
79 | do { |
80 | unsigned long *nums = frame_vector_pfns(vec); |
81 | |
82 | while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { |
83 | err = follow_pfn(vma, start, &nums[ret]); |
84 | if (err) { |
85 | if (ret == 0) |
86 | ret = err; |
87 | goto out; |
88 | } |
89 | start += PAGE_SIZE; |
90 | ret++; |
91 | } |
92 | /* |
93 | * We stop if we have enough pages or if VMA doesn't completely |
94 | * cover the tail page. |
95 | */ |
96 | if (ret >= nr_frames || start < vma->vm_end) |
97 | break; |
98 | vma = find_vma_intersection(mm, start, start + 1); |
99 | } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); |
100 | out: |
101 | if (locked) |
102 | up_read(&mm->mmap_sem); |
103 | if (!ret) |
104 | ret = -EFAULT; |
105 | if (ret > 0) |
106 | vec->nr_frames = ret; |
107 | return ret; |
108 | } |
109 | EXPORT_SYMBOL(get_vaddr_frames); |
110 | |
111 | /** |
112 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired |
113 | * them |
114 | * @vec: frame vector to put |
115 | * |
116 | * Drop references to pages if get_vaddr_frames() acquired them. We also |
117 | * invalidate the frame vector so that it is prepared for the next call into |
118 | * get_vaddr_frames(). |
119 | */ |
120 | void put_vaddr_frames(struct frame_vector *vec) |
121 | { |
122 | int i; |
123 | struct page **pages; |
124 | |
125 | if (!vec->got_ref) |
126 | goto out; |
127 | pages = frame_vector_pages(vec); |
128 | /* |
129 | * frame_vector_pages() might needed to do a conversion when |
130 | * get_vaddr_frames() got pages but vec was later converted to pfns. |
131 | * But it shouldn't really fail to convert pfns back... |
132 | */ |
133 | if (WARN_ON(IS_ERR(pages))) |
134 | goto out; |
135 | for (i = 0; i < vec->nr_frames; i++) |
136 | put_page(pages[i]); |
137 | vec->got_ref = false; |
138 | out: |
139 | vec->nr_frames = 0; |
140 | } |
141 | EXPORT_SYMBOL(put_vaddr_frames); |
142 | |
143 | /** |
144 | * frame_vector_to_pages - convert frame vector to contain page pointers |
145 | * @vec: frame vector to convert |
146 | * |
147 | * Convert @vec to contain array of page pointers. If the conversion is |
148 | * successful, return 0. Otherwise return an error. Note that we do not grab |
149 | * page references for the page structures. |
150 | */ |
151 | int frame_vector_to_pages(struct frame_vector *vec) |
152 | { |
153 | int i; |
154 | unsigned long *nums; |
155 | struct page **pages; |
156 | |
157 | if (!vec->is_pfns) |
158 | return 0; |
159 | nums = frame_vector_pfns(vec); |
160 | for (i = 0; i < vec->nr_frames; i++) |
161 | if (!pfn_valid(nums[i])) |
162 | return -EINVAL; |
163 | pages = (struct page **)nums; |
164 | for (i = 0; i < vec->nr_frames; i++) |
165 | pages[i] = pfn_to_page(nums[i]); |
166 | vec->is_pfns = false; |
167 | return 0; |
168 | } |
169 | EXPORT_SYMBOL(frame_vector_to_pages); |
170 | |
171 | /** |
172 | * frame_vector_to_pfns - convert frame vector to contain pfns |
173 | * @vec: frame vector to convert |
174 | * |
175 | * Convert @vec to contain array of pfns. |
176 | */ |
177 | void frame_vector_to_pfns(struct frame_vector *vec) |
178 | { |
179 | int i; |
180 | unsigned long *nums; |
181 | struct page **pages; |
182 | |
183 | if (vec->is_pfns) |
184 | return; |
185 | pages = (struct page **)(vec->ptrs); |
186 | nums = (unsigned long *)pages; |
187 | for (i = 0; i < vec->nr_frames; i++) |
188 | nums[i] = page_to_pfn(pages[i]); |
189 | vec->is_pfns = true; |
190 | } |
191 | EXPORT_SYMBOL(frame_vector_to_pfns); |
192 | |
193 | /** |
194 | * frame_vector_create() - allocate & initialize structure for pinned pfns |
195 | * @nr_frames: number of pfns slots we should reserve |
196 | * |
197 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns |
198 | * pfns. |
199 | */ |
200 | struct frame_vector *frame_vector_create(unsigned int nr_frames) |
201 | { |
202 | struct frame_vector *vec; |
203 | int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; |
204 | |
205 | if (WARN_ON_ONCE(nr_frames == 0)) |
206 | return NULL; |
207 | /* |
208 | * This is absurdly high. It's here just to avoid strange effects when |
209 | * arithmetics overflows. |
210 | */ |
211 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) |
212 | return NULL; |
213 | /* |
214 | * Avoid higher order allocations, use vmalloc instead. It should |
215 | * be rare anyway. |
216 | */ |
217 | if (size <= PAGE_SIZE) |
218 | vec = kmalloc(size, GFP_KERNEL); |
219 | else |
220 | vec = vmalloc(size); |
221 | if (!vec) |
222 | return NULL; |
223 | vec->nr_allocated = nr_frames; |
224 | vec->nr_frames = 0; |
225 | return vec; |
226 | } |
227 | EXPORT_SYMBOL(frame_vector_create); |
228 | |
229 | /** |
230 | * frame_vector_destroy() - free memory allocated to carry frame vector |
231 | * @vec: Frame vector to free |
232 | * |
233 | * Free structure allocated by frame_vector_create() to carry frames. |
234 | */ |
235 | void frame_vector_destroy(struct frame_vector *vec) |
236 | { |
237 | /* Make sure put_vaddr_frames() got called properly... */ |
238 | VM_BUG_ON(vec->nr_frames > 0); |
239 | kvfree(vec); |
240 | } |
241 | EXPORT_SYMBOL(frame_vector_destroy); |
242 |