blob: 60634dc53a885debd78878692f806c93883cbfff
1 | #include <linux/debugfs.h> |
2 | #include <linux/mm.h> |
3 | #include <linux/slab.h> |
4 | #include <linux/uaccess.h> |
5 | #include <linux/bootmem.h> |
6 | #include <linux/stacktrace.h> |
7 | #include <linux/page_owner.h> |
8 | #include <linux/jump_label.h> |
9 | #include <linux/migrate.h> |
10 | #include <linux/stackdepot.h> |
11 | #include <linux/seq_file.h> |
12 | |
13 | #include "internal.h" |
14 | |
15 | /* |
16 | * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) |
17 | * to use off stack temporal storage |
18 | */ |
19 | #define PAGE_OWNER_STACK_DEPTH (16) |
20 | |
21 | struct page_owner { |
22 | unsigned int order; |
23 | gfp_t gfp_mask; |
24 | int last_migrate_reason; |
25 | depot_stack_handle_t handle; |
26 | }; |
27 | |
28 | static bool page_owner_disabled = true; |
29 | DEFINE_STATIC_KEY_FALSE(page_owner_inited); |
30 | |
31 | static depot_stack_handle_t dummy_handle; |
32 | static depot_stack_handle_t failure_handle; |
33 | |
34 | static void init_early_allocated_pages(void); |
35 | |
36 | static int early_page_owner_param(char *buf) |
37 | { |
38 | if (!buf) |
39 | return -EINVAL; |
40 | |
41 | if (strcmp(buf, "on") == 0) |
42 | page_owner_disabled = false; |
43 | |
44 | return 0; |
45 | } |
46 | early_param("page_owner", early_page_owner_param); |
47 | |
48 | static bool need_page_owner(void) |
49 | { |
50 | if (page_owner_disabled) |
51 | return false; |
52 | |
53 | return true; |
54 | } |
55 | |
56 | static noinline void register_dummy_stack(void) |
57 | { |
58 | unsigned long entries[4]; |
59 | struct stack_trace dummy; |
60 | |
61 | dummy.nr_entries = 0; |
62 | dummy.max_entries = ARRAY_SIZE(entries); |
63 | dummy.entries = &entries[0]; |
64 | dummy.skip = 0; |
65 | |
66 | save_stack_trace(&dummy); |
67 | dummy_handle = depot_save_stack(&dummy, GFP_KERNEL); |
68 | } |
69 | |
70 | static noinline void register_failure_stack(void) |
71 | { |
72 | unsigned long entries[4]; |
73 | struct stack_trace failure; |
74 | |
75 | failure.nr_entries = 0; |
76 | failure.max_entries = ARRAY_SIZE(entries); |
77 | failure.entries = &entries[0]; |
78 | failure.skip = 0; |
79 | |
80 | save_stack_trace(&failure); |
81 | failure_handle = depot_save_stack(&failure, GFP_KERNEL); |
82 | } |
83 | |
84 | static void init_page_owner(void) |
85 | { |
86 | if (page_owner_disabled) |
87 | return; |
88 | |
89 | register_dummy_stack(); |
90 | register_failure_stack(); |
91 | static_branch_enable(&page_owner_inited); |
92 | init_early_allocated_pages(); |
93 | } |
94 | |
95 | struct page_ext_operations page_owner_ops = { |
96 | .size = sizeof(struct page_owner), |
97 | .need = need_page_owner, |
98 | .init = init_page_owner, |
99 | }; |
100 | |
101 | static inline struct page_owner *get_page_owner(struct page_ext *page_ext) |
102 | { |
103 | return (void *)page_ext + page_owner_ops.offset; |
104 | } |
105 | |
106 | void __reset_page_owner(struct page *page, unsigned int order) |
107 | { |
108 | int i; |
109 | struct page_ext *page_ext; |
110 | |
111 | for (i = 0; i < (1 << order); i++) { |
112 | page_ext = lookup_page_ext(page + i); |
113 | if (unlikely(!page_ext)) |
114 | continue; |
115 | __clear_bit(PAGE_EXT_OWNER, &page_ext->flags); |
116 | } |
117 | } |
118 | |
119 | static inline bool check_recursive_alloc(struct stack_trace *trace, |
120 | unsigned long ip) |
121 | { |
122 | int i, count; |
123 | |
124 | if (!trace->nr_entries) |
125 | return false; |
126 | |
127 | for (i = 0, count = 0; i < trace->nr_entries; i++) { |
128 | if (trace->entries[i] == ip && ++count == 2) |
129 | return true; |
130 | } |
131 | |
132 | return false; |
133 | } |
134 | |
135 | static noinline depot_stack_handle_t save_stack(gfp_t flags) |
136 | { |
137 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
138 | struct stack_trace trace = { |
139 | .nr_entries = 0, |
140 | .entries = entries, |
141 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
142 | .skip = 0 |
143 | }; |
144 | depot_stack_handle_t handle; |
145 | |
146 | save_stack_trace(&trace); |
147 | if (trace.nr_entries != 0 && |
148 | trace.entries[trace.nr_entries-1] == ULONG_MAX) |
149 | trace.nr_entries--; |
150 | |
151 | /* |
152 | * We need to check recursion here because our request to stackdepot |
153 | * could trigger memory allocation to save new entry. New memory |
154 | * allocation would reach here and call depot_save_stack() again |
155 | * if we don't catch it. There is still not enough memory in stackdepot |
156 | * so it would try to allocate memory again and loop forever. |
157 | */ |
158 | if (check_recursive_alloc(&trace, _RET_IP_)) |
159 | return dummy_handle; |
160 | |
161 | handle = depot_save_stack(&trace, flags); |
162 | if (!handle) |
163 | handle = failure_handle; |
164 | |
165 | return handle; |
166 | } |
167 | |
168 | noinline void __set_page_owner(struct page *page, unsigned int order, |
169 | gfp_t gfp_mask) |
170 | { |
171 | struct page_ext *page_ext = lookup_page_ext(page); |
172 | struct page_owner *page_owner; |
173 | |
174 | if (unlikely(!page_ext)) |
175 | return; |
176 | |
177 | page_owner = get_page_owner(page_ext); |
178 | page_owner->handle = save_stack(gfp_mask); |
179 | page_owner->order = order; |
180 | page_owner->gfp_mask = gfp_mask; |
181 | page_owner->last_migrate_reason = -1; |
182 | |
183 | __set_bit(PAGE_EXT_OWNER, &page_ext->flags); |
184 | } |
185 | |
186 | void __set_page_owner_migrate_reason(struct page *page, int reason) |
187 | { |
188 | struct page_ext *page_ext = lookup_page_ext(page); |
189 | struct page_owner *page_owner; |
190 | |
191 | if (unlikely(!page_ext)) |
192 | return; |
193 | |
194 | page_owner = get_page_owner(page_ext); |
195 | page_owner->last_migrate_reason = reason; |
196 | } |
197 | |
198 | void __split_page_owner(struct page *page, unsigned int order) |
199 | { |
200 | int i; |
201 | struct page_ext *page_ext = lookup_page_ext(page); |
202 | struct page_owner *page_owner; |
203 | |
204 | if (unlikely(!page_ext)) |
205 | return; |
206 | |
207 | page_owner = get_page_owner(page_ext); |
208 | page_owner->order = 0; |
209 | for (i = 1; i < (1 << order); i++) |
210 | __copy_page_owner(page, page + i); |
211 | } |
212 | |
213 | void __copy_page_owner(struct page *oldpage, struct page *newpage) |
214 | { |
215 | struct page_ext *old_ext = lookup_page_ext(oldpage); |
216 | struct page_ext *new_ext = lookup_page_ext(newpage); |
217 | struct page_owner *old_page_owner, *new_page_owner; |
218 | |
219 | if (unlikely(!old_ext || !new_ext)) |
220 | return; |
221 | |
222 | old_page_owner = get_page_owner(old_ext); |
223 | new_page_owner = get_page_owner(new_ext); |
224 | new_page_owner->order = old_page_owner->order; |
225 | new_page_owner->gfp_mask = old_page_owner->gfp_mask; |
226 | new_page_owner->last_migrate_reason = |
227 | old_page_owner->last_migrate_reason; |
228 | new_page_owner->handle = old_page_owner->handle; |
229 | |
230 | /* |
231 | * We don't clear the bit on the oldpage as it's going to be freed |
232 | * after migration. Until then, the info can be useful in case of |
233 | * a bug, and the overal stats will be off a bit only temporarily. |
234 | * Also, migrate_misplaced_transhuge_page() can still fail the |
235 | * migration and then we want the oldpage to retain the info. But |
236 | * in that case we also don't need to explicitly clear the info from |
237 | * the new page, which will be freed. |
238 | */ |
239 | __set_bit(PAGE_EXT_OWNER, &new_ext->flags); |
240 | } |
241 | |
242 | void pagetypeinfo_showmixedcount_print(struct seq_file *m, |
243 | pg_data_t *pgdat, struct zone *zone) |
244 | { |
245 | struct page *page; |
246 | struct page_ext *page_ext; |
247 | struct page_owner *page_owner; |
248 | unsigned long pfn = zone->zone_start_pfn, block_end_pfn; |
249 | unsigned long end_pfn = pfn + zone->spanned_pages; |
250 | unsigned long count[MIGRATE_TYPES] = { 0, }; |
251 | int pageblock_mt, page_mt; |
252 | int i; |
253 | |
254 | /* Scan block by block. First and last block may be incomplete */ |
255 | pfn = zone->zone_start_pfn; |
256 | |
257 | /* |
258 | * Walk the zone in pageblock_nr_pages steps. If a page block spans |
259 | * a zone boundary, it will be double counted between zones. This does |
260 | * not matter as the mixed block count will still be correct |
261 | */ |
262 | for (; pfn < end_pfn; ) { |
263 | if (!pfn_valid(pfn)) { |
264 | pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); |
265 | continue; |
266 | } |
267 | |
268 | block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); |
269 | block_end_pfn = min(block_end_pfn, end_pfn); |
270 | |
271 | page = pfn_to_page(pfn); |
272 | pageblock_mt = get_pageblock_migratetype(page); |
273 | |
274 | for (; pfn < block_end_pfn; pfn++) { |
275 | if (!pfn_valid_within(pfn)) |
276 | continue; |
277 | |
278 | page = pfn_to_page(pfn); |
279 | |
280 | if (page_zone(page) != zone) |
281 | continue; |
282 | |
283 | if (PageBuddy(page)) { |
284 | pfn += (1UL << page_order(page)) - 1; |
285 | continue; |
286 | } |
287 | |
288 | if (PageReserved(page)) |
289 | continue; |
290 | |
291 | page_ext = lookup_page_ext(page); |
292 | if (unlikely(!page_ext)) |
293 | continue; |
294 | |
295 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
296 | continue; |
297 | |
298 | page_owner = get_page_owner(page_ext); |
299 | page_mt = gfpflags_to_migratetype( |
300 | page_owner->gfp_mask); |
301 | if (pageblock_mt != page_mt) { |
302 | if (is_migrate_cma(pageblock_mt)) |
303 | count[MIGRATE_MOVABLE]++; |
304 | else |
305 | count[pageblock_mt]++; |
306 | |
307 | pfn = block_end_pfn; |
308 | break; |
309 | } |
310 | pfn += (1UL << page_owner->order) - 1; |
311 | } |
312 | } |
313 | |
314 | /* Print counts */ |
315 | seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); |
316 | for (i = 0; i < MIGRATE_TYPES; i++) |
317 | seq_printf(m, "%12lu ", count[i]); |
318 | seq_putc(m, '\n'); |
319 | } |
320 | |
321 | static ssize_t |
322 | print_page_owner(char __user *buf, size_t count, unsigned long pfn, |
323 | struct page *page, struct page_owner *page_owner, |
324 | depot_stack_handle_t handle) |
325 | { |
326 | int ret; |
327 | int pageblock_mt, page_mt; |
328 | char *kbuf; |
329 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
330 | struct stack_trace trace = { |
331 | .nr_entries = 0, |
332 | .entries = entries, |
333 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
334 | .skip = 0 |
335 | }; |
336 | |
337 | kbuf = kmalloc(count, GFP_KERNEL); |
338 | if (!kbuf) |
339 | return -ENOMEM; |
340 | |
341 | ret = snprintf(kbuf, count, |
342 | "Page allocated via order %u, mask %#x(%pGg)\n", |
343 | page_owner->order, page_owner->gfp_mask, |
344 | &page_owner->gfp_mask); |
345 | |
346 | if (ret >= count) |
347 | goto err; |
348 | |
349 | /* Print information relevant to grouping pages by mobility */ |
350 | pageblock_mt = get_pageblock_migratetype(page); |
351 | page_mt = gfpflags_to_migratetype(page_owner->gfp_mask); |
352 | ret += snprintf(kbuf + ret, count - ret, |
353 | "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", |
354 | pfn, |
355 | migratetype_names[page_mt], |
356 | pfn >> pageblock_order, |
357 | migratetype_names[pageblock_mt], |
358 | page->flags, &page->flags); |
359 | |
360 | if (ret >= count) |
361 | goto err; |
362 | |
363 | depot_fetch_stack(handle, &trace); |
364 | ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0); |
365 | if (ret >= count) |
366 | goto err; |
367 | |
368 | if (page_owner->last_migrate_reason != -1) { |
369 | ret += snprintf(kbuf + ret, count - ret, |
370 | "Page has been migrated, last migrate reason: %s\n", |
371 | migrate_reason_names[page_owner->last_migrate_reason]); |
372 | if (ret >= count) |
373 | goto err; |
374 | } |
375 | |
376 | ret += snprintf(kbuf + ret, count - ret, "\n"); |
377 | if (ret >= count) |
378 | goto err; |
379 | |
380 | if (copy_to_user(buf, kbuf, ret)) |
381 | ret = -EFAULT; |
382 | |
383 | kfree(kbuf); |
384 | return ret; |
385 | |
386 | err: |
387 | kfree(kbuf); |
388 | return -ENOMEM; |
389 | } |
390 | |
391 | void __dump_page_owner(struct page *page) |
392 | { |
393 | struct page_ext *page_ext = lookup_page_ext(page); |
394 | struct page_owner *page_owner; |
395 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
396 | struct stack_trace trace = { |
397 | .nr_entries = 0, |
398 | .entries = entries, |
399 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
400 | .skip = 0 |
401 | }; |
402 | depot_stack_handle_t handle; |
403 | gfp_t gfp_mask; |
404 | int mt; |
405 | |
406 | if (unlikely(!page_ext)) { |
407 | pr_alert("There is not page extension available.\n"); |
408 | return; |
409 | } |
410 | |
411 | page_owner = get_page_owner(page_ext); |
412 | gfp_mask = page_owner->gfp_mask; |
413 | mt = gfpflags_to_migratetype(gfp_mask); |
414 | |
415 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { |
416 | pr_alert("page_owner info is not active (free page?)\n"); |
417 | return; |
418 | } |
419 | |
420 | handle = READ_ONCE(page_owner->handle); |
421 | if (!handle) { |
422 | pr_alert("page_owner info is not active (free page?)\n"); |
423 | return; |
424 | } |
425 | |
426 | depot_fetch_stack(handle, &trace); |
427 | pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n", |
428 | page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask); |
429 | print_stack_trace(&trace, 0); |
430 | |
431 | if (page_owner->last_migrate_reason != -1) |
432 | pr_alert("page has been migrated, last migrate reason: %s\n", |
433 | migrate_reason_names[page_owner->last_migrate_reason]); |
434 | } |
435 | |
436 | static ssize_t |
437 | read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) |
438 | { |
439 | unsigned long pfn; |
440 | struct page *page; |
441 | struct page_ext *page_ext; |
442 | struct page_owner *page_owner; |
443 | depot_stack_handle_t handle; |
444 | |
445 | if (!static_branch_unlikely(&page_owner_inited)) |
446 | return -EINVAL; |
447 | |
448 | page = NULL; |
449 | pfn = min_low_pfn + *ppos; |
450 | |
451 | /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ |
452 | while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) |
453 | pfn++; |
454 | |
455 | drain_all_pages(NULL); |
456 | |
457 | /* Find an allocated page */ |
458 | for (; pfn < max_pfn; pfn++) { |
459 | /* |
460 | * If the new page is in a new MAX_ORDER_NR_PAGES area, |
461 | * validate the area as existing, skip it if not |
462 | */ |
463 | if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { |
464 | pfn += MAX_ORDER_NR_PAGES - 1; |
465 | continue; |
466 | } |
467 | |
468 | /* Check for holes within a MAX_ORDER area */ |
469 | if (!pfn_valid_within(pfn)) |
470 | continue; |
471 | |
472 | page = pfn_to_page(pfn); |
473 | if (PageBuddy(page)) { |
474 | unsigned long freepage_order = page_order_unsafe(page); |
475 | |
476 | if (freepage_order < MAX_ORDER) |
477 | pfn += (1UL << freepage_order) - 1; |
478 | continue; |
479 | } |
480 | |
481 | page_ext = lookup_page_ext(page); |
482 | if (unlikely(!page_ext)) |
483 | continue; |
484 | |
485 | /* |
486 | * Some pages could be missed by concurrent allocation or free, |
487 | * because we don't hold the zone lock. |
488 | */ |
489 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
490 | continue; |
491 | |
492 | page_owner = get_page_owner(page_ext); |
493 | |
494 | /* |
495 | * Access to page_ext->handle isn't synchronous so we should |
496 | * be careful to access it. |
497 | */ |
498 | handle = READ_ONCE(page_owner->handle); |
499 | if (!handle) |
500 | continue; |
501 | |
502 | /* Record the next PFN to read in the file offset */ |
503 | *ppos = (pfn - min_low_pfn) + 1; |
504 | |
505 | return print_page_owner(buf, count, pfn, page, |
506 | page_owner, handle); |
507 | } |
508 | |
509 | return 0; |
510 | } |
511 | |
512 | static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) |
513 | { |
514 | struct page *page; |
515 | struct page_ext *page_ext; |
516 | unsigned long pfn = zone->zone_start_pfn, block_end_pfn; |
517 | unsigned long end_pfn = pfn + zone->spanned_pages; |
518 | unsigned long count = 0; |
519 | |
520 | /* Scan block by block. First and last block may be incomplete */ |
521 | pfn = zone->zone_start_pfn; |
522 | |
523 | /* |
524 | * Walk the zone in pageblock_nr_pages steps. If a page block spans |
525 | * a zone boundary, it will be double counted between zones. This does |
526 | * not matter as the mixed block count will still be correct |
527 | */ |
528 | for (; pfn < end_pfn; ) { |
529 | if (!pfn_valid(pfn)) { |
530 | pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); |
531 | continue; |
532 | } |
533 | |
534 | block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); |
535 | block_end_pfn = min(block_end_pfn, end_pfn); |
536 | |
537 | page = pfn_to_page(pfn); |
538 | |
539 | for (; pfn < block_end_pfn; pfn++) { |
540 | if (!pfn_valid_within(pfn)) |
541 | continue; |
542 | |
543 | page = pfn_to_page(pfn); |
544 | |
545 | if (page_zone(page) != zone) |
546 | continue; |
547 | |
548 | /* |
549 | * We are safe to check buddy flag and order, because |
550 | * this is init stage and only single thread runs. |
551 | */ |
552 | if (PageBuddy(page)) { |
553 | pfn += (1UL << page_order(page)) - 1; |
554 | continue; |
555 | } |
556 | |
557 | if (PageReserved(page)) |
558 | continue; |
559 | |
560 | page_ext = lookup_page_ext(page); |
561 | if (unlikely(!page_ext)) |
562 | continue; |
563 | |
564 | /* Maybe overraping zone */ |
565 | if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
566 | continue; |
567 | |
568 | /* Found early allocated page */ |
569 | set_page_owner(page, 0, 0); |
570 | count++; |
571 | } |
572 | } |
573 | |
574 | pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", |
575 | pgdat->node_id, zone->name, count); |
576 | } |
577 | |
578 | static void init_zones_in_node(pg_data_t *pgdat) |
579 | { |
580 | struct zone *zone; |
581 | struct zone *node_zones = pgdat->node_zones; |
582 | unsigned long flags; |
583 | |
584 | for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { |
585 | if (!populated_zone(zone)) |
586 | continue; |
587 | |
588 | spin_lock_irqsave(&zone->lock, flags); |
589 | init_pages_in_zone(pgdat, zone); |
590 | spin_unlock_irqrestore(&zone->lock, flags); |
591 | } |
592 | } |
593 | |
594 | static void init_early_allocated_pages(void) |
595 | { |
596 | pg_data_t *pgdat; |
597 | |
598 | drain_all_pages(NULL); |
599 | for_each_online_pgdat(pgdat) |
600 | init_zones_in_node(pgdat); |
601 | } |
602 | |
603 | static const struct file_operations proc_page_owner_operations = { |
604 | .read = read_page_owner, |
605 | }; |
606 | |
607 | static int __init pageowner_init(void) |
608 | { |
609 | struct dentry *dentry; |
610 | |
611 | if (!static_branch_unlikely(&page_owner_inited)) { |
612 | pr_info("page_owner is disabled\n"); |
613 | return 0; |
614 | } |
615 | |
616 | dentry = debugfs_create_file("page_owner", S_IRUSR, NULL, |
617 | NULL, &proc_page_owner_operations); |
618 | if (IS_ERR(dentry)) |
619 | return PTR_ERR(dentry); |
620 | |
621 | return 0; |
622 | } |
623 | late_initcall(pageowner_init) |
624 |