blob: fd2c9acbcc19856f4b53e28d1acd9f5d1cdae165
1 | #include <linux/kdebug.h> |
2 | #include <linux/kprobes.h> |
3 | #include <linux/export.h> |
4 | #include <linux/notifier.h> |
5 | #include <linux/rcupdate.h> |
6 | #include <linux/vmalloc.h> |
7 | #include <linux/reboot.h> |
8 | |
9 | /* |
10 | * Notifier list for kernel code which wants to be called |
11 | * at shutdown. This is used to stop any idling DMA operations |
12 | * and the like. |
13 | */ |
14 | BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); |
15 | |
16 | /* |
17 | * Notifier chain core routines. The exported routines below |
18 | * are layered on top of these, with appropriate locking added. |
19 | */ |
20 | |
21 | static int notifier_chain_register(struct notifier_block **nl, |
22 | struct notifier_block *n) |
23 | { |
24 | while ((*nl) != NULL) { |
25 | if (n->priority > (*nl)->priority) |
26 | break; |
27 | nl = &((*nl)->next); |
28 | } |
29 | n->next = *nl; |
30 | rcu_assign_pointer(*nl, n); |
31 | return 0; |
32 | } |
33 | |
34 | static int notifier_chain_cond_register(struct notifier_block **nl, |
35 | struct notifier_block *n) |
36 | { |
37 | while ((*nl) != NULL) { |
38 | if ((*nl) == n) |
39 | return 0; |
40 | if (n->priority > (*nl)->priority) |
41 | break; |
42 | nl = &((*nl)->next); |
43 | } |
44 | n->next = *nl; |
45 | rcu_assign_pointer(*nl, n); |
46 | return 0; |
47 | } |
48 | |
49 | static int notifier_chain_unregister(struct notifier_block **nl, |
50 | struct notifier_block *n) |
51 | { |
52 | while ((*nl) != NULL) { |
53 | if ((*nl) == n) { |
54 | rcu_assign_pointer(*nl, n->next); |
55 | return 0; |
56 | } |
57 | nl = &((*nl)->next); |
58 | } |
59 | return -ENOENT; |
60 | } |
61 | |
62 | /** |
63 | * notifier_call_chain - Informs the registered notifiers about an event. |
64 | * @nl: Pointer to head of the blocking notifier chain |
65 | * @val: Value passed unmodified to notifier function |
66 | * @v: Pointer passed unmodified to notifier function |
67 | * @nr_to_call: Number of notifier functions to be called. Don't care |
68 | * value of this parameter is -1. |
69 | * @nr_calls: Records the number of notifications sent. Don't care |
70 | * value of this field is NULL. |
71 | * @returns: notifier_call_chain returns the value returned by the |
72 | * last notifier function called. |
73 | */ |
74 | static int notifier_call_chain(struct notifier_block **nl, |
75 | unsigned long val, void *v, |
76 | int nr_to_call, int *nr_calls) |
77 | { |
78 | int ret = NOTIFY_DONE; |
79 | struct notifier_block *nb, *next_nb; |
80 | |
81 | nb = rcu_dereference_raw(*nl); |
82 | |
83 | while (nb && nr_to_call) { |
84 | next_nb = rcu_dereference_raw(nb->next); |
85 | |
86 | #ifdef CONFIG_DEBUG_NOTIFIERS |
87 | if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { |
88 | WARN(1, "Invalid notifier called!"); |
89 | nb = next_nb; |
90 | continue; |
91 | } |
92 | #endif |
93 | ret = nb->notifier_call(nb, val, v); |
94 | |
95 | if (nr_calls) |
96 | (*nr_calls)++; |
97 | |
98 | if ((ret & NOTIFY_STOP_MASK) == NOTIFY_STOP_MASK) |
99 | break; |
100 | nb = next_nb; |
101 | nr_to_call--; |
102 | } |
103 | return ret; |
104 | } |
105 | NOKPROBE_SYMBOL(notifier_call_chain); |
106 | |
107 | /* |
108 | * Atomic notifier chain routines. Registration and unregistration |
109 | * use a spinlock, and call_chain is synchronized by RCU (no locks). |
110 | */ |
111 | |
112 | /** |
113 | * atomic_notifier_chain_register - Add notifier to an atomic notifier chain |
114 | * @nh: Pointer to head of the atomic notifier chain |
115 | * @n: New entry in notifier chain |
116 | * |
117 | * Adds a notifier to an atomic notifier chain. |
118 | * |
119 | * Currently always returns zero. |
120 | */ |
121 | int atomic_notifier_chain_register(struct atomic_notifier_head *nh, |
122 | struct notifier_block *n) |
123 | { |
124 | unsigned long flags; |
125 | int ret; |
126 | |
127 | spin_lock_irqsave(&nh->lock, flags); |
128 | ret = notifier_chain_register(&nh->head, n); |
129 | spin_unlock_irqrestore(&nh->lock, flags); |
130 | return ret; |
131 | } |
132 | EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); |
133 | |
134 | /** |
135 | * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain |
136 | * @nh: Pointer to head of the atomic notifier chain |
137 | * @n: Entry to remove from notifier chain |
138 | * |
139 | * Removes a notifier from an atomic notifier chain. |
140 | * |
141 | * Returns zero on success or %-ENOENT on failure. |
142 | */ |
143 | int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, |
144 | struct notifier_block *n) |
145 | { |
146 | unsigned long flags; |
147 | int ret; |
148 | |
149 | spin_lock_irqsave(&nh->lock, flags); |
150 | ret = notifier_chain_unregister(&nh->head, n); |
151 | spin_unlock_irqrestore(&nh->lock, flags); |
152 | synchronize_rcu(); |
153 | return ret; |
154 | } |
155 | EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); |
156 | |
157 | /** |
158 | * __atomic_notifier_call_chain - Call functions in an atomic notifier chain |
159 | * @nh: Pointer to head of the atomic notifier chain |
160 | * @val: Value passed unmodified to notifier function |
161 | * @v: Pointer passed unmodified to notifier function |
162 | * @nr_to_call: See the comment for notifier_call_chain. |
163 | * @nr_calls: See the comment for notifier_call_chain. |
164 | * |
165 | * Calls each function in a notifier chain in turn. The functions |
166 | * run in an atomic context, so they must not block. |
167 | * This routine uses RCU to synchronize with changes to the chain. |
168 | * |
169 | * If the return value of the notifier can be and'ed |
170 | * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() |
171 | * will return immediately, with the return value of |
172 | * the notifier function which halted execution. |
173 | * Otherwise the return value is the return value |
174 | * of the last notifier function called. |
175 | */ |
176 | int __atomic_notifier_call_chain(struct atomic_notifier_head *nh, |
177 | unsigned long val, void *v, |
178 | int nr_to_call, int *nr_calls) |
179 | { |
180 | int ret; |
181 | |
182 | rcu_read_lock(); |
183 | ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
184 | rcu_read_unlock(); |
185 | return ret; |
186 | } |
187 | EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain); |
188 | NOKPROBE_SYMBOL(__atomic_notifier_call_chain); |
189 | |
190 | int atomic_notifier_call_chain(struct atomic_notifier_head *nh, |
191 | unsigned long val, void *v) |
192 | { |
193 | return __atomic_notifier_call_chain(nh, val, v, -1, NULL); |
194 | } |
195 | EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); |
196 | NOKPROBE_SYMBOL(atomic_notifier_call_chain); |
197 | |
198 | /* |
199 | * Blocking notifier chain routines. All access to the chain is |
200 | * synchronized by an rwsem. |
201 | */ |
202 | |
203 | /** |
204 | * blocking_notifier_chain_register - Add notifier to a blocking notifier chain |
205 | * @nh: Pointer to head of the blocking notifier chain |
206 | * @n: New entry in notifier chain |
207 | * |
208 | * Adds a notifier to a blocking notifier chain. |
209 | * Must be called in process context. |
210 | * |
211 | * Currently always returns zero. |
212 | */ |
213 | int blocking_notifier_chain_register(struct blocking_notifier_head *nh, |
214 | struct notifier_block *n) |
215 | { |
216 | int ret; |
217 | |
218 | /* |
219 | * This code gets used during boot-up, when task switching is |
220 | * not yet working and interrupts must remain disabled. At |
221 | * such times we must not call down_write(). |
222 | */ |
223 | if (unlikely(system_state == SYSTEM_BOOTING)) |
224 | return notifier_chain_register(&nh->head, n); |
225 | |
226 | down_write(&nh->rwsem); |
227 | ret = notifier_chain_register(&nh->head, n); |
228 | up_write(&nh->rwsem); |
229 | return ret; |
230 | } |
231 | EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); |
232 | |
233 | /** |
234 | * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain |
235 | * @nh: Pointer to head of the blocking notifier chain |
236 | * @n: New entry in notifier chain |
237 | * |
238 | * Adds a notifier to a blocking notifier chain, only if not already |
239 | * present in the chain. |
240 | * Must be called in process context. |
241 | * |
242 | * Currently always returns zero. |
243 | */ |
244 | int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh, |
245 | struct notifier_block *n) |
246 | { |
247 | int ret; |
248 | |
249 | down_write(&nh->rwsem); |
250 | ret = notifier_chain_cond_register(&nh->head, n); |
251 | up_write(&nh->rwsem); |
252 | return ret; |
253 | } |
254 | EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register); |
255 | |
256 | /** |
257 | * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain |
258 | * @nh: Pointer to head of the blocking notifier chain |
259 | * @n: Entry to remove from notifier chain |
260 | * |
261 | * Removes a notifier from a blocking notifier chain. |
262 | * Must be called from process context. |
263 | * |
264 | * Returns zero on success or %-ENOENT on failure. |
265 | */ |
266 | int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, |
267 | struct notifier_block *n) |
268 | { |
269 | int ret; |
270 | |
271 | /* |
272 | * This code gets used during boot-up, when task switching is |
273 | * not yet working and interrupts must remain disabled. At |
274 | * such times we must not call down_write(). |
275 | */ |
276 | if (unlikely(system_state == SYSTEM_BOOTING)) |
277 | return notifier_chain_unregister(&nh->head, n); |
278 | |
279 | down_write(&nh->rwsem); |
280 | ret = notifier_chain_unregister(&nh->head, n); |
281 | up_write(&nh->rwsem); |
282 | return ret; |
283 | } |
284 | EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); |
285 | |
286 | /** |
287 | * __blocking_notifier_call_chain - Call functions in a blocking notifier chain |
288 | * @nh: Pointer to head of the blocking notifier chain |
289 | * @val: Value passed unmodified to notifier function |
290 | * @v: Pointer passed unmodified to notifier function |
291 | * @nr_to_call: See comment for notifier_call_chain. |
292 | * @nr_calls: See comment for notifier_call_chain. |
293 | * |
294 | * Calls each function in a notifier chain in turn. The functions |
295 | * run in a process context, so they are allowed to block. |
296 | * |
297 | * If the return value of the notifier can be and'ed |
298 | * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() |
299 | * will return immediately, with the return value of |
300 | * the notifier function which halted execution. |
301 | * Otherwise the return value is the return value |
302 | * of the last notifier function called. |
303 | */ |
304 | int __blocking_notifier_call_chain(struct blocking_notifier_head *nh, |
305 | unsigned long val, void *v, |
306 | int nr_to_call, int *nr_calls) |
307 | { |
308 | int ret = NOTIFY_DONE; |
309 | |
310 | /* |
311 | * We check the head outside the lock, but if this access is |
312 | * racy then it does not matter what the result of the test |
313 | * is, we re-check the list after having taken the lock anyway: |
314 | */ |
315 | if (rcu_access_pointer(nh->head)) { |
316 | down_read(&nh->rwsem); |
317 | ret = notifier_call_chain(&nh->head, val, v, nr_to_call, |
318 | nr_calls); |
319 | up_read(&nh->rwsem); |
320 | } |
321 | return ret; |
322 | } |
323 | EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain); |
324 | |
325 | int blocking_notifier_call_chain(struct blocking_notifier_head *nh, |
326 | unsigned long val, void *v) |
327 | { |
328 | return __blocking_notifier_call_chain(nh, val, v, -1, NULL); |
329 | } |
330 | EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); |
331 | |
332 | /* |
333 | * Raw notifier chain routines. There is no protection; |
334 | * the caller must provide it. Use at your own risk! |
335 | */ |
336 | |
337 | /** |
338 | * raw_notifier_chain_register - Add notifier to a raw notifier chain |
339 | * @nh: Pointer to head of the raw notifier chain |
340 | * @n: New entry in notifier chain |
341 | * |
342 | * Adds a notifier to a raw notifier chain. |
343 | * All locking must be provided by the caller. |
344 | * |
345 | * Currently always returns zero. |
346 | */ |
347 | int raw_notifier_chain_register(struct raw_notifier_head *nh, |
348 | struct notifier_block *n) |
349 | { |
350 | return notifier_chain_register(&nh->head, n); |
351 | } |
352 | EXPORT_SYMBOL_GPL(raw_notifier_chain_register); |
353 | |
354 | /** |
355 | * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain |
356 | * @nh: Pointer to head of the raw notifier chain |
357 | * @n: Entry to remove from notifier chain |
358 | * |
359 | * Removes a notifier from a raw notifier chain. |
360 | * All locking must be provided by the caller. |
361 | * |
362 | * Returns zero on success or %-ENOENT on failure. |
363 | */ |
364 | int raw_notifier_chain_unregister(struct raw_notifier_head *nh, |
365 | struct notifier_block *n) |
366 | { |
367 | return notifier_chain_unregister(&nh->head, n); |
368 | } |
369 | EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); |
370 | |
371 | /** |
372 | * __raw_notifier_call_chain - Call functions in a raw notifier chain |
373 | * @nh: Pointer to head of the raw notifier chain |
374 | * @val: Value passed unmodified to notifier function |
375 | * @v: Pointer passed unmodified to notifier function |
376 | * @nr_to_call: See comment for notifier_call_chain. |
377 | * @nr_calls: See comment for notifier_call_chain |
378 | * |
379 | * Calls each function in a notifier chain in turn. The functions |
380 | * run in an undefined context. |
381 | * All locking must be provided by the caller. |
382 | * |
383 | * If the return value of the notifier can be and'ed |
384 | * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() |
385 | * will return immediately, with the return value of |
386 | * the notifier function which halted execution. |
387 | * Otherwise the return value is the return value |
388 | * of the last notifier function called. |
389 | */ |
390 | int __raw_notifier_call_chain(struct raw_notifier_head *nh, |
391 | unsigned long val, void *v, |
392 | int nr_to_call, int *nr_calls) |
393 | { |
394 | return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
395 | } |
396 | EXPORT_SYMBOL_GPL(__raw_notifier_call_chain); |
397 | |
398 | int raw_notifier_call_chain(struct raw_notifier_head *nh, |
399 | unsigned long val, void *v) |
400 | { |
401 | return __raw_notifier_call_chain(nh, val, v, -1, NULL); |
402 | } |
403 | EXPORT_SYMBOL_GPL(raw_notifier_call_chain); |
404 | |
405 | #ifdef CONFIG_SRCU |
406 | /* |
407 | * SRCU notifier chain routines. Registration and unregistration |
408 | * use a mutex, and call_chain is synchronized by SRCU (no locks). |
409 | */ |
410 | |
411 | /** |
412 | * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain |
413 | * @nh: Pointer to head of the SRCU notifier chain |
414 | * @n: New entry in notifier chain |
415 | * |
416 | * Adds a notifier to an SRCU notifier chain. |
417 | * Must be called in process context. |
418 | * |
419 | * Currently always returns zero. |
420 | */ |
421 | int srcu_notifier_chain_register(struct srcu_notifier_head *nh, |
422 | struct notifier_block *n) |
423 | { |
424 | int ret; |
425 | |
426 | /* |
427 | * This code gets used during boot-up, when task switching is |
428 | * not yet working and interrupts must remain disabled. At |
429 | * such times we must not call mutex_lock(). |
430 | */ |
431 | if (unlikely(system_state == SYSTEM_BOOTING)) |
432 | return notifier_chain_register(&nh->head, n); |
433 | |
434 | mutex_lock(&nh->mutex); |
435 | ret = notifier_chain_register(&nh->head, n); |
436 | mutex_unlock(&nh->mutex); |
437 | return ret; |
438 | } |
439 | EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); |
440 | |
441 | /** |
442 | * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain |
443 | * @nh: Pointer to head of the SRCU notifier chain |
444 | * @n: Entry to remove from notifier chain |
445 | * |
446 | * Removes a notifier from an SRCU notifier chain. |
447 | * Must be called from process context. |
448 | * |
449 | * Returns zero on success or %-ENOENT on failure. |
450 | */ |
451 | int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, |
452 | struct notifier_block *n) |
453 | { |
454 | int ret; |
455 | |
456 | /* |
457 | * This code gets used during boot-up, when task switching is |
458 | * not yet working and interrupts must remain disabled. At |
459 | * such times we must not call mutex_lock(). |
460 | */ |
461 | if (unlikely(system_state == SYSTEM_BOOTING)) |
462 | return notifier_chain_unregister(&nh->head, n); |
463 | |
464 | mutex_lock(&nh->mutex); |
465 | ret = notifier_chain_unregister(&nh->head, n); |
466 | mutex_unlock(&nh->mutex); |
467 | synchronize_srcu(&nh->srcu); |
468 | return ret; |
469 | } |
470 | EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); |
471 | |
472 | /** |
473 | * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain |
474 | * @nh: Pointer to head of the SRCU notifier chain |
475 | * @val: Value passed unmodified to notifier function |
476 | * @v: Pointer passed unmodified to notifier function |
477 | * @nr_to_call: See comment for notifier_call_chain. |
478 | * @nr_calls: See comment for notifier_call_chain |
479 | * |
480 | * Calls each function in a notifier chain in turn. The functions |
481 | * run in a process context, so they are allowed to block. |
482 | * |
483 | * If the return value of the notifier can be and'ed |
484 | * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() |
485 | * will return immediately, with the return value of |
486 | * the notifier function which halted execution. |
487 | * Otherwise the return value is the return value |
488 | * of the last notifier function called. |
489 | */ |
490 | int __srcu_notifier_call_chain(struct srcu_notifier_head *nh, |
491 | unsigned long val, void *v, |
492 | int nr_to_call, int *nr_calls) |
493 | { |
494 | int ret; |
495 | int idx; |
496 | |
497 | idx = srcu_read_lock(&nh->srcu); |
498 | ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
499 | srcu_read_unlock(&nh->srcu, idx); |
500 | return ret; |
501 | } |
502 | EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain); |
503 | |
504 | int srcu_notifier_call_chain(struct srcu_notifier_head *nh, |
505 | unsigned long val, void *v) |
506 | { |
507 | return __srcu_notifier_call_chain(nh, val, v, -1, NULL); |
508 | } |
509 | EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); |
510 | |
511 | /** |
512 | * srcu_init_notifier_head - Initialize an SRCU notifier head |
513 | * @nh: Pointer to head of the srcu notifier chain |
514 | * |
515 | * Unlike other sorts of notifier heads, SRCU notifier heads require |
516 | * dynamic initialization. Be sure to call this routine before |
517 | * calling any of the other SRCU notifier routines for this head. |
518 | * |
519 | * If an SRCU notifier head is deallocated, it must first be cleaned |
520 | * up by calling srcu_cleanup_notifier_head(). Otherwise the head's |
521 | * per-cpu data (used by the SRCU mechanism) will leak. |
522 | */ |
523 | void srcu_init_notifier_head(struct srcu_notifier_head *nh) |
524 | { |
525 | mutex_init(&nh->mutex); |
526 | if (init_srcu_struct(&nh->srcu) < 0) |
527 | BUG(); |
528 | nh->head = NULL; |
529 | } |
530 | EXPORT_SYMBOL_GPL(srcu_init_notifier_head); |
531 | |
532 | #endif /* CONFIG_SRCU */ |
533 | |
534 | static ATOMIC_NOTIFIER_HEAD(die_chain); |
535 | |
536 | int notrace notify_die(enum die_val val, const char *str, |
537 | struct pt_regs *regs, long err, int trap, int sig) |
538 | { |
539 | struct die_args args = { |
540 | .regs = regs, |
541 | .str = str, |
542 | .err = err, |
543 | .trapnr = trap, |
544 | .signr = sig, |
545 | |
546 | }; |
547 | RCU_LOCKDEP_WARN(!rcu_is_watching(), |
548 | "notify_die called but RCU thinks we're quiescent"); |
549 | return atomic_notifier_call_chain(&die_chain, val, &args); |
550 | } |
551 | NOKPROBE_SYMBOL(notify_die); |
552 | |
553 | int register_die_notifier(struct notifier_block *nb) |
554 | { |
555 | vmalloc_sync_all(); |
556 | return atomic_notifier_chain_register(&die_chain, nb); |
557 | } |
558 | EXPORT_SYMBOL_GPL(register_die_notifier); |
559 | |
560 | int unregister_die_notifier(struct notifier_block *nb) |
561 | { |
562 | return atomic_notifier_chain_unregister(&die_chain, nb); |
563 | } |
564 | EXPORT_SYMBOL_GPL(unregister_die_notifier); |
565 |