blob: 6063581f705c48b97a78a087d8c67127af9c8d68
1 | /* |
2 | * Linux VM pressure |
3 | * |
4 | * Copyright 2012 Linaro Ltd. |
5 | * Anton Vorontsov <anton.vorontsov@linaro.org> |
6 | * |
7 | * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, |
8 | * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify it |
11 | * under the terms of the GNU General Public License version 2 as published |
12 | * by the Free Software Foundation. |
13 | */ |
14 | |
15 | #include <linux/cgroup.h> |
16 | #include <linux/fs.h> |
17 | #include <linux/log2.h> |
18 | #include <linux/sched.h> |
19 | #include <linux/mm.h> |
20 | #include <linux/vmstat.h> |
21 | #include <linux/eventfd.h> |
22 | #include <linux/slab.h> |
23 | #include <linux/swap.h> |
24 | #include <linux/printk.h> |
25 | #include <linux/vmpressure.h> |
26 | |
27 | /* |
28 | * The window size (vmpressure_win) is the number of scanned pages before |
29 | * we try to analyze scanned/reclaimed ratio. So the window is used as a |
30 | * rate-limit tunable for the "low" level notification, and also for |
31 | * averaging the ratio for medium/critical levels. Using small window |
32 | * sizes can cause lot of false positives, but too big window size will |
33 | * delay the notifications. |
34 | * |
35 | * As the vmscan reclaimer logic works with chunks which are multiple of |
36 | * SWAP_CLUSTER_MAX, it makes sense to use it for the window size as well. |
37 | * |
38 | * TODO: Make the window size depend on machine size, as we do for vmstat |
39 | * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). |
40 | */ |
41 | static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; |
42 | |
43 | /* |
44 | * These thresholds are used when we account memory pressure through |
45 | * scanned/reclaimed ratio. The current values were chosen empirically. In |
46 | * essence, they are percents: the higher the value, the more number |
47 | * unsuccessful reclaims there were. |
48 | */ |
49 | static const unsigned int vmpressure_level_med = 60; |
50 | static const unsigned int vmpressure_level_critical = 95; |
51 | |
52 | /* |
53 | * When there are too little pages left to scan, vmpressure() may miss the |
54 | * critical pressure as number of pages will be less than "window size". |
55 | * However, in that case the vmscan priority will raise fast as the |
56 | * reclaimer will try to scan LRUs more deeply. |
57 | * |
58 | * The vmscan logic considers these special priorities: |
59 | * |
60 | * prio == DEF_PRIORITY (12): reclaimer starts with that value |
61 | * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed |
62 | * prio == 0 : close to OOM, kernel scans every page in an lru |
63 | * |
64 | * Any value in this range is acceptable for this tunable (i.e. from 12 to |
65 | * 0). Current value for the vmpressure_level_critical_prio is chosen |
66 | * empirically, but the number, in essence, means that we consider |
67 | * critical level when scanning depth is ~10% of the lru size (vmscan |
68 | * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one |
69 | * eights). |
70 | */ |
71 | static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); |
72 | |
73 | static struct vmpressure *work_to_vmpressure(struct work_struct *work) |
74 | { |
75 | return container_of(work, struct vmpressure, work); |
76 | } |
77 | |
78 | static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) |
79 | { |
80 | struct cgroup_subsys_state *css = vmpressure_to_css(vmpr); |
81 | struct mem_cgroup *memcg = mem_cgroup_from_css(css); |
82 | |
83 | memcg = parent_mem_cgroup(memcg); |
84 | if (!memcg) |
85 | return NULL; |
86 | return memcg_to_vmpressure(memcg); |
87 | } |
88 | |
89 | enum vmpressure_levels { |
90 | VMPRESSURE_LOW = 0, |
91 | VMPRESSURE_MEDIUM, |
92 | VMPRESSURE_CRITICAL, |
93 | VMPRESSURE_NUM_LEVELS, |
94 | }; |
95 | |
96 | static const char * const vmpressure_str_levels[] = { |
97 | [VMPRESSURE_LOW] = "low", |
98 | [VMPRESSURE_MEDIUM] = "medium", |
99 | [VMPRESSURE_CRITICAL] = "critical", |
100 | }; |
101 | |
102 | static enum vmpressure_levels vmpressure_level(unsigned long pressure) |
103 | { |
104 | if (pressure >= vmpressure_level_critical) |
105 | return VMPRESSURE_CRITICAL; |
106 | else if (pressure >= vmpressure_level_med) |
107 | return VMPRESSURE_MEDIUM; |
108 | return VMPRESSURE_LOW; |
109 | } |
110 | |
111 | static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, |
112 | unsigned long reclaimed) |
113 | { |
114 | unsigned long scale = scanned + reclaimed; |
115 | unsigned long pressure = 0; |
116 | |
117 | /* |
118 | * reclaimed can be greater than scanned in cases |
119 | * like THP, where the scanned is 1 and reclaimed |
120 | * could be 512 |
121 | */ |
122 | if (reclaimed >= scanned) |
123 | goto out; |
124 | /* |
125 | * We calculate the ratio (in percents) of how many pages were |
126 | * scanned vs. reclaimed in a given time frame (window). Note that |
127 | * time is in VM reclaimer's "ticks", i.e. number of pages |
128 | * scanned. This makes it possible to set desired reaction time |
129 | * and serves as a ratelimit. |
130 | */ |
131 | pressure = scale - (reclaimed * scale / scanned); |
132 | pressure = pressure * 100 / scale; |
133 | |
134 | out: |
135 | pr_debug("%s: %3lu (s: %lu r: %lu)\n", __func__, pressure, |
136 | scanned, reclaimed); |
137 | |
138 | return vmpressure_level(pressure); |
139 | } |
140 | |
141 | struct vmpressure_event { |
142 | struct eventfd_ctx *efd; |
143 | enum vmpressure_levels level; |
144 | struct list_head node; |
145 | }; |
146 | |
147 | static bool vmpressure_event(struct vmpressure *vmpr, |
148 | enum vmpressure_levels level) |
149 | { |
150 | struct vmpressure_event *ev; |
151 | bool signalled = false; |
152 | |
153 | mutex_lock(&vmpr->events_lock); |
154 | |
155 | list_for_each_entry(ev, &vmpr->events, node) { |
156 | if (level >= ev->level) { |
157 | eventfd_signal(ev->efd, 1); |
158 | signalled = true; |
159 | } |
160 | } |
161 | |
162 | mutex_unlock(&vmpr->events_lock); |
163 | |
164 | return signalled; |
165 | } |
166 | |
167 | static void vmpressure_work_fn(struct work_struct *work) |
168 | { |
169 | struct vmpressure *vmpr = work_to_vmpressure(work); |
170 | unsigned long scanned; |
171 | unsigned long reclaimed; |
172 | enum vmpressure_levels level; |
173 | |
174 | spin_lock(&vmpr->sr_lock); |
175 | /* |
176 | * Several contexts might be calling vmpressure(), so it is |
177 | * possible that the work was rescheduled again before the old |
178 | * work context cleared the counters. In that case we will run |
179 | * just after the old work returns, but then scanned might be zero |
180 | * here. No need for any locks here since we don't care if |
181 | * vmpr->reclaimed is in sync. |
182 | */ |
183 | scanned = vmpr->tree_scanned; |
184 | if (!scanned) { |
185 | spin_unlock(&vmpr->sr_lock); |
186 | return; |
187 | } |
188 | |
189 | reclaimed = vmpr->tree_reclaimed; |
190 | vmpr->tree_scanned = 0; |
191 | vmpr->tree_reclaimed = 0; |
192 | spin_unlock(&vmpr->sr_lock); |
193 | |
194 | level = vmpressure_calc_level(scanned, reclaimed); |
195 | |
196 | do { |
197 | if (vmpressure_event(vmpr, level)) |
198 | break; |
199 | /* |
200 | * If not handled, propagate the event upward into the |
201 | * hierarchy. |
202 | */ |
203 | } while ((vmpr = vmpressure_parent(vmpr))); |
204 | } |
205 | |
206 | /** |
207 | * vmpressure() - Account memory pressure through scanned/reclaimed ratio |
208 | * @gfp: reclaimer's gfp mask |
209 | * @memcg: cgroup memory controller handle |
210 | * @tree: legacy subtree mode |
211 | * @scanned: number of pages scanned |
212 | * @reclaimed: number of pages reclaimed |
213 | * |
214 | * This function should be called from the vmscan reclaim path to account |
215 | * "instantaneous" memory pressure (scanned/reclaimed ratio). The raw |
216 | * pressure index is then further refined and averaged over time. |
217 | * |
218 | * If @tree is set, vmpressure is in traditional userspace reporting |
219 | * mode: @memcg is considered the pressure root and userspace is |
220 | * notified of the entire subtree's reclaim efficiency. |
221 | * |
222 | * If @tree is not set, reclaim efficiency is recorded for @memcg, and |
223 | * only in-kernel users are notified. |
224 | * |
225 | * This function does not return any value. |
226 | */ |
227 | void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree, |
228 | unsigned long scanned, unsigned long reclaimed) |
229 | { |
230 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
231 | |
232 | /* |
233 | * Here we only want to account pressure that userland is able to |
234 | * help us with. For example, suppose that DMA zone is under |
235 | * pressure; if we notify userland about that kind of pressure, |
236 | * then it will be mostly a waste as it will trigger unnecessary |
237 | * freeing of memory by userland (since userland is more likely to |
238 | * have HIGHMEM/MOVABLE pages instead of the DMA fallback). That |
239 | * is why we include only movable, highmem and FS/IO pages. |
240 | * Indirect reclaim (kswapd) sets sc->gfp_mask to GFP_KERNEL, so |
241 | * we account it too. |
242 | */ |
243 | if (!(gfp & (__GFP_HIGHMEM | __GFP_MOVABLE | __GFP_IO | __GFP_FS))) |
244 | return; |
245 | |
246 | /* |
247 | * If we got here with no pages scanned, then that is an indicator |
248 | * that reclaimer was unable to find any shrinkable LRUs at the |
249 | * current scanning depth. But it does not mean that we should |
250 | * report the critical pressure, yet. If the scanning priority |
251 | * (scanning depth) goes too high (deep), we will be notified |
252 | * through vmpressure_prio(). But so far, keep calm. |
253 | */ |
254 | if (!scanned) |
255 | return; |
256 | |
257 | if (tree) { |
258 | spin_lock(&vmpr->sr_lock); |
259 | scanned = vmpr->tree_scanned += scanned; |
260 | vmpr->tree_reclaimed += reclaimed; |
261 | spin_unlock(&vmpr->sr_lock); |
262 | |
263 | if (scanned < vmpressure_win) |
264 | return; |
265 | schedule_work(&vmpr->work); |
266 | } else { |
267 | enum vmpressure_levels level; |
268 | |
269 | /* For now, no users for root-level efficiency */ |
270 | if (!memcg || memcg == root_mem_cgroup) |
271 | return; |
272 | |
273 | spin_lock(&vmpr->sr_lock); |
274 | scanned = vmpr->scanned += scanned; |
275 | reclaimed = vmpr->reclaimed += reclaimed; |
276 | if (scanned < vmpressure_win) { |
277 | spin_unlock(&vmpr->sr_lock); |
278 | return; |
279 | } |
280 | vmpr->scanned = vmpr->reclaimed = 0; |
281 | spin_unlock(&vmpr->sr_lock); |
282 | |
283 | level = vmpressure_calc_level(scanned, reclaimed); |
284 | |
285 | if (level > VMPRESSURE_LOW) { |
286 | /* |
287 | * Let the socket buffer allocator know that |
288 | * we are having trouble reclaiming LRU pages. |
289 | * |
290 | * For hysteresis keep the pressure state |
291 | * asserted for a second in which subsequent |
292 | * pressure events can occur. |
293 | */ |
294 | memcg->socket_pressure = jiffies + HZ; |
295 | } |
296 | } |
297 | } |
298 | |
299 | /** |
300 | * vmpressure_prio() - Account memory pressure through reclaimer priority level |
301 | * @gfp: reclaimer's gfp mask |
302 | * @memcg: cgroup memory controller handle |
303 | * @prio: reclaimer's priority |
304 | * |
305 | * This function should be called from the reclaim path every time when |
306 | * the vmscan's reclaiming priority (scanning depth) changes. |
307 | * |
308 | * This function does not return any value. |
309 | */ |
310 | void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) |
311 | { |
312 | /* |
313 | * We only use prio for accounting critical level. For more info |
314 | * see comment for vmpressure_level_critical_prio variable above. |
315 | */ |
316 | if (prio > vmpressure_level_critical_prio) |
317 | return; |
318 | |
319 | /* |
320 | * OK, the prio is below the threshold, updating vmpressure |
321 | * information before shrinker dives into long shrinking of long |
322 | * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 |
323 | * to the vmpressure() basically means that we signal 'critical' |
324 | * level. |
325 | */ |
326 | vmpressure(gfp, memcg, true, vmpressure_win, 0); |
327 | } |
328 | |
329 | /** |
330 | * vmpressure_register_event() - Bind vmpressure notifications to an eventfd |
331 | * @memcg: memcg that is interested in vmpressure notifications |
332 | * @eventfd: eventfd context to link notifications with |
333 | * @args: event arguments (used to set up a pressure level threshold) |
334 | * |
335 | * This function associates eventfd context with the vmpressure |
336 | * infrastructure, so that the notifications will be delivered to the |
337 | * @eventfd. The @args parameter is a string that denotes pressure level |
338 | * threshold (one of vmpressure_str_levels, i.e. "low", "medium", or |
339 | * "critical"). |
340 | * |
341 | * To be used as memcg event method. |
342 | */ |
343 | int vmpressure_register_event(struct mem_cgroup *memcg, |
344 | struct eventfd_ctx *eventfd, const char *args) |
345 | { |
346 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
347 | struct vmpressure_event *ev; |
348 | int level; |
349 | |
350 | for (level = 0; level < VMPRESSURE_NUM_LEVELS; level++) { |
351 | if (!strcmp(vmpressure_str_levels[level], args)) |
352 | break; |
353 | } |
354 | |
355 | if (level >= VMPRESSURE_NUM_LEVELS) |
356 | return -EINVAL; |
357 | |
358 | ev = kzalloc(sizeof(*ev), GFP_KERNEL); |
359 | if (!ev) |
360 | return -ENOMEM; |
361 | |
362 | ev->efd = eventfd; |
363 | ev->level = level; |
364 | |
365 | mutex_lock(&vmpr->events_lock); |
366 | list_add(&ev->node, &vmpr->events); |
367 | mutex_unlock(&vmpr->events_lock); |
368 | |
369 | return 0; |
370 | } |
371 | |
372 | /** |
373 | * vmpressure_unregister_event() - Unbind eventfd from vmpressure |
374 | * @memcg: memcg handle |
375 | * @eventfd: eventfd context that was used to link vmpressure with the @cg |
376 | * |
377 | * This function does internal manipulations to detach the @eventfd from |
378 | * the vmpressure notifications, and then frees internal resources |
379 | * associated with the @eventfd (but the @eventfd itself is not freed). |
380 | * |
381 | * To be used as memcg event method. |
382 | */ |
383 | void vmpressure_unregister_event(struct mem_cgroup *memcg, |
384 | struct eventfd_ctx *eventfd) |
385 | { |
386 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
387 | struct vmpressure_event *ev; |
388 | |
389 | mutex_lock(&vmpr->events_lock); |
390 | list_for_each_entry(ev, &vmpr->events, node) { |
391 | if (ev->efd != eventfd) |
392 | continue; |
393 | list_del(&ev->node); |
394 | kfree(ev); |
395 | break; |
396 | } |
397 | mutex_unlock(&vmpr->events_lock); |
398 | } |
399 | |
400 | /** |
401 | * vmpressure_init() - Initialize vmpressure control structure |
402 | * @vmpr: Structure to be initialized |
403 | * |
404 | * This function should be called on every allocated vmpressure structure |
405 | * before any usage. |
406 | */ |
407 | void vmpressure_init(struct vmpressure *vmpr) |
408 | { |
409 | spin_lock_init(&vmpr->sr_lock); |
410 | mutex_init(&vmpr->events_lock); |
411 | INIT_LIST_HEAD(&vmpr->events); |
412 | INIT_WORK(&vmpr->work, vmpressure_work_fn); |
413 | } |
414 | |
415 | /** |
416 | * vmpressure_cleanup() - shuts down vmpressure control structure |
417 | * @vmpr: Structure to be cleaned up |
418 | * |
419 | * This function should be called before the structure in which it is |
420 | * embedded is cleaned up. |
421 | */ |
422 | void vmpressure_cleanup(struct vmpressure *vmpr) |
423 | { |
424 | /* |
425 | * Make sure there is no pending work before eventfd infrastructure |
426 | * goes away. |
427 | */ |
428 | flush_work(&vmpr->work); |
429 | } |
430 |