Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/kdebug.h>
3 : #include <linux/kprobes.h>
4 : #include <linux/export.h>
5 : #include <linux/notifier.h>
6 : #include <linux/rcupdate.h>
7 : #include <linux/vmalloc.h>
8 : #include <linux/reboot.h>
9 :
10 : #define CREATE_TRACE_POINTS
11 : #include <trace/events/notifier.h>
12 :
13 : /*
14 : * Notifier list for kernel code which wants to be called
15 : * at shutdown. This is used to stop any idling DMA operations
16 : * and the like.
17 : */
18 : BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
19 :
20 : /*
21 : * Notifier chain core routines. The exported routines below
22 : * are layered on top of these, with appropriate locking added.
23 : */
24 :
25 8 : static int notifier_chain_register(struct notifier_block **nl,
26 : struct notifier_block *n,
27 : bool unique_priority)
28 : {
29 18 : while ((*nl) != NULL) {
30 3 : if (unlikely((*nl) == n)) {
31 0 : WARN(1, "notifier callback %ps already registered",
32 : n->notifier_call);
33 0 : return -EEXIST;
34 : }
35 3 : if (n->priority > (*nl)->priority)
36 : break;
37 2 : if (n->priority == (*nl)->priority && unique_priority)
38 : return -EBUSY;
39 2 : nl = &((*nl)->next);
40 : }
41 8 : n->next = *nl;
42 8 : rcu_assign_pointer(*nl, n);
43 8 : trace_notifier_register((void *)n->notifier_call);
44 8 : return 0;
45 : }
46 :
47 : static int notifier_chain_unregister(struct notifier_block **nl,
48 : struct notifier_block *n)
49 : {
50 0 : while ((*nl) != NULL) {
51 0 : if ((*nl) == n) {
52 0 : rcu_assign_pointer(*nl, n->next);
53 0 : trace_notifier_unregister((void *)n->notifier_call);
54 : return 0;
55 : }
56 0 : nl = &((*nl)->next);
57 : }
58 : return -ENOENT;
59 : }
60 :
61 : /**
62 : * notifier_call_chain - Informs the registered notifiers about an event.
63 : * @nl: Pointer to head of the blocking notifier chain
64 : * @val: Value passed unmodified to notifier function
65 : * @v: Pointer passed unmodified to notifier function
66 : * @nr_to_call: Number of notifier functions to be called. Don't care
67 : * value of this parameter is -1.
68 : * @nr_calls: Records the number of notifications sent. Don't care
69 : * value of this field is NULL.
70 : * Return: notifier_call_chain returns the value returned by the
71 : * last notifier function called.
72 : */
73 : static int notifier_call_chain(struct notifier_block **nl,
74 : unsigned long val, void *v,
75 : int nr_to_call, int *nr_calls)
76 : {
77 2944 : int ret = NOTIFY_DONE;
78 : struct notifier_block *nb, *next_nb;
79 :
80 2944 : nb = rcu_dereference_raw(*nl);
81 :
82 2946 : while (nb && nr_to_call) {
83 2 : next_nb = rcu_dereference_raw(nb->next);
84 :
85 : #ifdef CONFIG_DEBUG_NOTIFIERS
86 : if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
87 : WARN(1, "Invalid notifier called!");
88 : nb = next_nb;
89 : continue;
90 : }
91 : #endif
92 2 : trace_notifier_run((void *)nb->notifier_call);
93 2 : ret = nb->notifier_call(nb, val, v);
94 :
95 : if (nr_calls)
96 0 : (*nr_calls)++;
97 :
98 2 : if (ret & NOTIFY_STOP_MASK)
99 : break;
100 2 : nb = next_nb;
101 2 : nr_to_call--;
102 : }
103 : return ret;
104 : }
105 : NOKPROBE_SYMBOL(notifier_call_chain);
106 :
107 : /**
108 : * notifier_call_chain_robust - Inform the registered notifiers about an event
109 : * and rollback on error.
110 : * @nl: Pointer to head of the blocking notifier chain
111 : * @val_up: Value passed unmodified to the notifier function
112 : * @val_down: Value passed unmodified to the notifier function when recovering
113 : * from an error on @val_up
114 : * @v: Pointer passed unmodified to the notifier function
115 : *
116 : * NOTE: It is important the @nl chain doesn't change between the two
117 : * invocations of notifier_call_chain() such that we visit the
118 : * exact same notifier callbacks; this rules out any RCU usage.
119 : *
120 : * Return: the return value of the @val_up call.
121 : */
122 0 : static int notifier_call_chain_robust(struct notifier_block **nl,
123 : unsigned long val_up, unsigned long val_down,
124 : void *v)
125 : {
126 0 : int ret, nr = 0;
127 :
128 0 : ret = notifier_call_chain(nl, val_up, v, -1, &nr);
129 0 : if (ret & NOTIFY_STOP_MASK)
130 0 : notifier_call_chain(nl, val_down, v, nr-1, NULL);
131 :
132 0 : return ret;
133 : }
134 :
135 : /*
136 : * Atomic notifier chain routines. Registration and unregistration
137 : * use a spinlock, and call_chain is synchronized by RCU (no locks).
138 : */
139 :
140 : /**
141 : * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
142 : * @nh: Pointer to head of the atomic notifier chain
143 : * @n: New entry in notifier chain
144 : *
145 : * Adds a notifier to an atomic notifier chain.
146 : *
147 : * Returns 0 on success, %-EEXIST on error.
148 : */
149 2 : int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
150 : struct notifier_block *n)
151 : {
152 : unsigned long flags;
153 : int ret;
154 :
155 2 : spin_lock_irqsave(&nh->lock, flags);
156 2 : ret = notifier_chain_register(&nh->head, n, false);
157 4 : spin_unlock_irqrestore(&nh->lock, flags);
158 2 : return ret;
159 : }
160 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
161 :
162 : /**
163 : * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
164 : * @nh: Pointer to head of the atomic notifier chain
165 : * @n: New entry in notifier chain
166 : *
167 : * Adds a notifier to an atomic notifier chain if there is no other
168 : * notifier registered using the same priority.
169 : *
170 : * Returns 0 on success, %-EEXIST or %-EBUSY on error.
171 : */
172 0 : int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
173 : struct notifier_block *n)
174 : {
175 : unsigned long flags;
176 : int ret;
177 :
178 0 : spin_lock_irqsave(&nh->lock, flags);
179 0 : ret = notifier_chain_register(&nh->head, n, true);
180 0 : spin_unlock_irqrestore(&nh->lock, flags);
181 0 : return ret;
182 : }
183 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
184 :
185 : /**
186 : * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
187 : * @nh: Pointer to head of the atomic notifier chain
188 : * @n: Entry to remove from notifier chain
189 : *
190 : * Removes a notifier from an atomic notifier chain.
191 : *
192 : * Returns zero on success or %-ENOENT on failure.
193 : */
194 0 : int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
195 : struct notifier_block *n)
196 : {
197 : unsigned long flags;
198 : int ret;
199 :
200 0 : spin_lock_irqsave(&nh->lock, flags);
201 0 : ret = notifier_chain_unregister(&nh->head, n);
202 0 : spin_unlock_irqrestore(&nh->lock, flags);
203 0 : synchronize_rcu();
204 0 : return ret;
205 : }
206 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
207 :
208 : /**
209 : * atomic_notifier_call_chain - Call functions in an atomic notifier chain
210 : * @nh: Pointer to head of the atomic notifier chain
211 : * @val: Value passed unmodified to notifier function
212 : * @v: Pointer passed unmodified to notifier function
213 : *
214 : * Calls each function in a notifier chain in turn. The functions
215 : * run in an atomic context, so they must not block.
216 : * This routine uses RCU to synchronize with changes to the chain.
217 : *
218 : * If the return value of the notifier can be and'ed
219 : * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
220 : * will return immediately, with the return value of
221 : * the notifier function which halted execution.
222 : * Otherwise the return value is the return value
223 : * of the last notifier function called.
224 : */
225 1 : int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
226 : unsigned long val, void *v)
227 : {
228 : int ret;
229 :
230 : rcu_read_lock();
231 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
232 : rcu_read_unlock();
233 :
234 1 : return ret;
235 : }
236 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
237 : NOKPROBE_SYMBOL(atomic_notifier_call_chain);
238 :
239 : /**
240 : * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
241 : * @nh: Pointer to head of the atomic notifier chain
242 : *
243 : * Checks whether notifier chain is empty.
244 : *
245 : * Returns true is notifier chain is empty, false otherwise.
246 : */
247 0 : bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
248 : {
249 0 : return !rcu_access_pointer(nh->head);
250 : }
251 :
252 : /*
253 : * Blocking notifier chain routines. All access to the chain is
254 : * synchronized by an rwsem.
255 : */
256 :
257 6 : static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
258 : struct notifier_block *n,
259 : bool unique_priority)
260 : {
261 : int ret;
262 :
263 : /*
264 : * This code gets used during boot-up, when task switching is
265 : * not yet working and interrupts must remain disabled. At
266 : * such times we must not call down_write().
267 : */
268 6 : if (unlikely(system_state == SYSTEM_BOOTING))
269 1 : return notifier_chain_register(&nh->head, n, unique_priority);
270 :
271 5 : down_write(&nh->rwsem);
272 5 : ret = notifier_chain_register(&nh->head, n, unique_priority);
273 5 : up_write(&nh->rwsem);
274 5 : return ret;
275 : }
276 :
277 : /**
278 : * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
279 : * @nh: Pointer to head of the blocking notifier chain
280 : * @n: New entry in notifier chain
281 : *
282 : * Adds a notifier to a blocking notifier chain.
283 : * Must be called in process context.
284 : *
285 : * Returns 0 on success, %-EEXIST on error.
286 : */
287 6 : int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
288 : struct notifier_block *n)
289 : {
290 6 : return __blocking_notifier_chain_register(nh, n, false);
291 : }
292 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
293 :
294 : /**
295 : * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
296 : * @nh: Pointer to head of the blocking notifier chain
297 : * @n: New entry in notifier chain
298 : *
299 : * Adds a notifier to an blocking notifier chain if there is no other
300 : * notifier registered using the same priority.
301 : *
302 : * Returns 0 on success, %-EEXIST or %-EBUSY on error.
303 : */
304 0 : int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
305 : struct notifier_block *n)
306 : {
307 0 : return __blocking_notifier_chain_register(nh, n, true);
308 : }
309 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
310 :
311 : /**
312 : * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
313 : * @nh: Pointer to head of the blocking notifier chain
314 : * @n: Entry to remove from notifier chain
315 : *
316 : * Removes a notifier from a blocking notifier chain.
317 : * Must be called from process context.
318 : *
319 : * Returns zero on success or %-ENOENT on failure.
320 : */
321 0 : int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
322 : struct notifier_block *n)
323 : {
324 : int ret;
325 :
326 : /*
327 : * This code gets used during boot-up, when task switching is
328 : * not yet working and interrupts must remain disabled. At
329 : * such times we must not call down_write().
330 : */
331 0 : if (unlikely(system_state == SYSTEM_BOOTING))
332 0 : return notifier_chain_unregister(&nh->head, n);
333 :
334 0 : down_write(&nh->rwsem);
335 0 : ret = notifier_chain_unregister(&nh->head, n);
336 0 : up_write(&nh->rwsem);
337 0 : return ret;
338 : }
339 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
340 :
341 0 : int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
342 : unsigned long val_up, unsigned long val_down, void *v)
343 : {
344 0 : int ret = NOTIFY_DONE;
345 :
346 : /*
347 : * We check the head outside the lock, but if this access is
348 : * racy then it does not matter what the result of the test
349 : * is, we re-check the list after having taken the lock anyway:
350 : */
351 0 : if (rcu_access_pointer(nh->head)) {
352 0 : down_read(&nh->rwsem);
353 0 : ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
354 0 : up_read(&nh->rwsem);
355 : }
356 0 : return ret;
357 : }
358 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
359 :
360 : /**
361 : * blocking_notifier_call_chain - Call functions in a blocking notifier chain
362 : * @nh: Pointer to head of the blocking notifier chain
363 : * @val: Value passed unmodified to notifier function
364 : * @v: Pointer passed unmodified to notifier function
365 : *
366 : * Calls each function in a notifier chain in turn. The functions
367 : * run in a process context, so they are allowed to block.
368 : *
369 : * If the return value of the notifier can be and'ed
370 : * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
371 : * will return immediately, with the return value of
372 : * the notifier function which halted execution.
373 : * Otherwise the return value is the return value
374 : * of the last notifier function called.
375 : */
376 159 : int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
377 : unsigned long val, void *v)
378 : {
379 159 : int ret = NOTIFY_DONE;
380 :
381 : /*
382 : * We check the head outside the lock, but if this access is
383 : * racy then it does not matter what the result of the test
384 : * is, we re-check the list after having taken the lock anyway:
385 : */
386 159 : if (rcu_access_pointer(nh->head)) {
387 1 : down_read(&nh->rwsem);
388 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
389 1 : up_read(&nh->rwsem);
390 : }
391 159 : return ret;
392 : }
393 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
394 :
395 : /*
396 : * Raw notifier chain routines. There is no protection;
397 : * the caller must provide it. Use at your own risk!
398 : */
399 :
400 : /**
401 : * raw_notifier_chain_register - Add notifier to a raw notifier chain
402 : * @nh: Pointer to head of the raw notifier chain
403 : * @n: New entry in notifier chain
404 : *
405 : * Adds a notifier to a raw notifier chain.
406 : * All locking must be provided by the caller.
407 : *
408 : * Returns 0 on success, %-EEXIST on error.
409 : */
410 0 : int raw_notifier_chain_register(struct raw_notifier_head *nh,
411 : struct notifier_block *n)
412 : {
413 0 : return notifier_chain_register(&nh->head, n, false);
414 : }
415 : EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
416 :
417 : /**
418 : * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
419 : * @nh: Pointer to head of the raw notifier chain
420 : * @n: Entry to remove from notifier chain
421 : *
422 : * Removes a notifier from a raw notifier chain.
423 : * All locking must be provided by the caller.
424 : *
425 : * Returns zero on success or %-ENOENT on failure.
426 : */
427 0 : int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
428 : struct notifier_block *n)
429 : {
430 0 : return notifier_chain_unregister(&nh->head, n);
431 : }
432 : EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
433 :
434 0 : int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
435 : unsigned long val_up, unsigned long val_down, void *v)
436 : {
437 0 : return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
438 : }
439 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
440 :
441 : /**
442 : * raw_notifier_call_chain - Call functions in a raw notifier chain
443 : * @nh: Pointer to head of the raw notifier chain
444 : * @val: Value passed unmodified to notifier function
445 : * @v: Pointer passed unmodified to notifier function
446 : *
447 : * Calls each function in a notifier chain in turn. The functions
448 : * run in an undefined context.
449 : * All locking must be provided by the caller.
450 : *
451 : * If the return value of the notifier can be and'ed
452 : * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
453 : * will return immediately, with the return value of
454 : * the notifier function which halted execution.
455 : * Otherwise the return value is the return value
456 : * of the last notifier function called.
457 : */
458 2942 : int raw_notifier_call_chain(struct raw_notifier_head *nh,
459 : unsigned long val, void *v)
460 : {
461 5884 : return notifier_call_chain(&nh->head, val, v, -1, NULL);
462 : }
463 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
464 :
465 : /*
466 : * SRCU notifier chain routines. Registration and unregistration
467 : * use a mutex, and call_chain is synchronized by SRCU (no locks).
468 : */
469 :
470 : /**
471 : * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
472 : * @nh: Pointer to head of the SRCU notifier chain
473 : * @n: New entry in notifier chain
474 : *
475 : * Adds a notifier to an SRCU notifier chain.
476 : * Must be called in process context.
477 : *
478 : * Returns 0 on success, %-EEXIST on error.
479 : */
480 0 : int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
481 : struct notifier_block *n)
482 : {
483 : int ret;
484 :
485 : /*
486 : * This code gets used during boot-up, when task switching is
487 : * not yet working and interrupts must remain disabled. At
488 : * such times we must not call mutex_lock().
489 : */
490 0 : if (unlikely(system_state == SYSTEM_BOOTING))
491 0 : return notifier_chain_register(&nh->head, n, false);
492 :
493 0 : mutex_lock(&nh->mutex);
494 0 : ret = notifier_chain_register(&nh->head, n, false);
495 0 : mutex_unlock(&nh->mutex);
496 0 : return ret;
497 : }
498 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
499 :
500 : /**
501 : * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
502 : * @nh: Pointer to head of the SRCU notifier chain
503 : * @n: Entry to remove from notifier chain
504 : *
505 : * Removes a notifier from an SRCU notifier chain.
506 : * Must be called from process context.
507 : *
508 : * Returns zero on success or %-ENOENT on failure.
509 : */
510 0 : int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
511 : struct notifier_block *n)
512 : {
513 : int ret;
514 :
515 : /*
516 : * This code gets used during boot-up, when task switching is
517 : * not yet working and interrupts must remain disabled. At
518 : * such times we must not call mutex_lock().
519 : */
520 0 : if (unlikely(system_state == SYSTEM_BOOTING))
521 0 : return notifier_chain_unregister(&nh->head, n);
522 :
523 0 : mutex_lock(&nh->mutex);
524 0 : ret = notifier_chain_unregister(&nh->head, n);
525 0 : mutex_unlock(&nh->mutex);
526 0 : synchronize_srcu(&nh->srcu);
527 0 : return ret;
528 : }
529 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
530 :
531 : /**
532 : * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
533 : * @nh: Pointer to head of the SRCU notifier chain
534 : * @val: Value passed unmodified to notifier function
535 : * @v: Pointer passed unmodified to notifier function
536 : *
537 : * Calls each function in a notifier chain in turn. The functions
538 : * run in a process context, so they are allowed to block.
539 : *
540 : * If the return value of the notifier can be and'ed
541 : * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
542 : * will return immediately, with the return value of
543 : * the notifier function which halted execution.
544 : * Otherwise the return value is the return value
545 : * of the last notifier function called.
546 : */
547 0 : int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
548 : unsigned long val, void *v)
549 : {
550 : int ret;
551 : int idx;
552 :
553 0 : idx = srcu_read_lock(&nh->srcu);
554 0 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
555 0 : srcu_read_unlock(&nh->srcu, idx);
556 0 : return ret;
557 : }
558 : EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
559 :
560 : /**
561 : * srcu_init_notifier_head - Initialize an SRCU notifier head
562 : * @nh: Pointer to head of the srcu notifier chain
563 : *
564 : * Unlike other sorts of notifier heads, SRCU notifier heads require
565 : * dynamic initialization. Be sure to call this routine before
566 : * calling any of the other SRCU notifier routines for this head.
567 : *
568 : * If an SRCU notifier head is deallocated, it must first be cleaned
569 : * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
570 : * per-cpu data (used by the SRCU mechanism) will leak.
571 : */
572 1 : void srcu_init_notifier_head(struct srcu_notifier_head *nh)
573 : {
574 1 : mutex_init(&nh->mutex);
575 1 : if (init_srcu_struct(&nh->srcu) < 0)
576 0 : BUG();
577 1 : nh->head = NULL;
578 1 : }
579 : EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
580 :
581 : static ATOMIC_NOTIFIER_HEAD(die_chain);
582 :
583 0 : int notrace notify_die(enum die_val val, const char *str,
584 : struct pt_regs *regs, long err, int trap, int sig)
585 : {
586 0 : struct die_args args = {
587 : .regs = regs,
588 : .str = str,
589 : .err = err,
590 : .trapnr = trap,
591 : .signr = sig,
592 :
593 : };
594 : RCU_LOCKDEP_WARN(!rcu_is_watching(),
595 : "notify_die called but RCU thinks we're quiescent");
596 0 : return atomic_notifier_call_chain(&die_chain, val, &args);
597 : }
598 : NOKPROBE_SYMBOL(notify_die);
599 :
600 0 : int register_die_notifier(struct notifier_block *nb)
601 : {
602 0 : return atomic_notifier_chain_register(&die_chain, nb);
603 : }
604 : EXPORT_SYMBOL_GPL(register_die_notifier);
605 :
606 0 : int unregister_die_notifier(struct notifier_block *nb)
607 : {
608 0 : return atomic_notifier_chain_unregister(&die_chain, nb);
609 : }
610 : EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
