Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <linux/kdebug.h>
3 : #include <linux/kprobes.h>
4 : #include <linux/export.h>
5 : #include <linux/notifier.h>
6 : #include <linux/rcupdate.h>
7 : #include <linux/vmalloc.h>
8 : #include <linux/reboot.h>
9 :
10 : /*
11 : * Notifier list for kernel code which wants to be called
12 : * at shutdown. This is used to stop any idling DMA operations
13 : * and the like.
14 : */
15 : BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16 :
17 : /*
18 : * Notifier chain core routines. The exported routines below
19 : * are layered on top of these, with appropriate locking added.
20 : */
21 :
22 8 : static int notifier_chain_register(struct notifier_block **nl,
23 : struct notifier_block *n,
24 : bool unique_priority)
25 : {
26 18 : while ((*nl) != NULL) {
27 3 : if (unlikely((*nl) == n)) {
28 0 : WARN(1, "notifier callback %ps already registered",
29 : n->notifier_call);
30 0 : return -EEXIST;
31 : }
32 3 : if (n->priority > (*nl)->priority)
33 : break;
34 2 : if (n->priority == (*nl)->priority && unique_priority)
35 : return -EBUSY;
36 2 : nl = &((*nl)->next);
37 : }
38 8 : n->next = *nl;
39 8 : rcu_assign_pointer(*nl, n);
40 8 : return 0;
41 : }
42 :
43 : static int notifier_chain_unregister(struct notifier_block **nl,
44 : struct notifier_block *n)
45 : {
46 0 : while ((*nl) != NULL) {
47 0 : if ((*nl) == n) {
48 0 : rcu_assign_pointer(*nl, n->next);
49 : return 0;
50 : }
51 0 : nl = &((*nl)->next);
52 : }
53 : return -ENOENT;
54 : }
55 :
56 : /**
57 : * notifier_call_chain - Informs the registered notifiers about an event.
58 : * @nl: Pointer to head of the blocking notifier chain
59 : * @val: Value passed unmodified to notifier function
60 : * @v: Pointer passed unmodified to notifier function
61 : * @nr_to_call: Number of notifier functions to be called. Don't care
62 : * value of this parameter is -1.
63 : * @nr_calls: Records the number of notifications sent. Don't care
64 : * value of this field is NULL.
65 : * Return: notifier_call_chain returns the value returned by the
66 : * last notifier function called.
67 : */
68 : static int notifier_call_chain(struct notifier_block **nl,
69 : unsigned long val, void *v,
70 : int nr_to_call, int *nr_calls)
71 : {
72 2751 : int ret = NOTIFY_DONE;
73 : struct notifier_block *nb, *next_nb;
74 :
75 2751 : nb = rcu_dereference_raw(*nl);
76 :
77 2753 : while (nb && nr_to_call) {
78 2 : next_nb = rcu_dereference_raw(nb->next);
79 :
80 : #ifdef CONFIG_DEBUG_NOTIFIERS
81 : if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
82 : WARN(1, "Invalid notifier called!");
83 : nb = next_nb;
84 : continue;
85 : }
86 : #endif
87 2 : ret = nb->notifier_call(nb, val, v);
88 :
89 : if (nr_calls)
90 0 : (*nr_calls)++;
91 :
92 2 : if (ret & NOTIFY_STOP_MASK)
93 : break;
94 2 : nb = next_nb;
95 2 : nr_to_call--;
96 : }
97 : return ret;
98 : }
99 : NOKPROBE_SYMBOL(notifier_call_chain);
100 :
101 : /**
102 : * notifier_call_chain_robust - Inform the registered notifiers about an event
103 : * and rollback on error.
104 : * @nl: Pointer to head of the blocking notifier chain
105 : * @val_up: Value passed unmodified to the notifier function
106 : * @val_down: Value passed unmodified to the notifier function when recovering
107 : * from an error on @val_up
108 : * @v: Pointer passed unmodified to the notifier function
109 : *
110 : * NOTE: It is important the @nl chain doesn't change between the two
111 : * invocations of notifier_call_chain() such that we visit the
112 : * exact same notifier callbacks; this rules out any RCU usage.
113 : *
114 : * Return: the return value of the @val_up call.
115 : */
116 0 : static int notifier_call_chain_robust(struct notifier_block **nl,
117 : unsigned long val_up, unsigned long val_down,
118 : void *v)
119 : {
120 0 : int ret, nr = 0;
121 :
122 0 : ret = notifier_call_chain(nl, val_up, v, -1, &nr);
123 0 : if (ret & NOTIFY_STOP_MASK)
124 0 : notifier_call_chain(nl, val_down, v, nr-1, NULL);
125 :
126 0 : return ret;
127 : }
128 :
129 : /*
130 : * Atomic notifier chain routines. Registration and unregistration
131 : * use a spinlock, and call_chain is synchronized by RCU (no locks).
132 : */
133 :
134 : /**
135 : * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
136 : * @nh: Pointer to head of the atomic notifier chain
137 : * @n: New entry in notifier chain
138 : *
139 : * Adds a notifier to an atomic notifier chain.
140 : *
141 : * Returns 0 on success, %-EEXIST on error.
142 : */
143 2 : int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
144 : struct notifier_block *n)
145 : {
146 : unsigned long flags;
147 : int ret;
148 :
149 2 : spin_lock_irqsave(&nh->lock, flags);
150 2 : ret = notifier_chain_register(&nh->head, n, false);
151 4 : spin_unlock_irqrestore(&nh->lock, flags);
152 2 : return ret;
153 : }
154 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
155 :
156 : /**
157 : * atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
158 : * @nh: Pointer to head of the atomic notifier chain
159 : * @n: New entry in notifier chain
160 : *
161 : * Adds a notifier to an atomic notifier chain if there is no other
162 : * notifier registered using the same priority.
163 : *
164 : * Returns 0 on success, %-EEXIST or %-EBUSY on error.
165 : */
166 0 : int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
167 : struct notifier_block *n)
168 : {
169 : unsigned long flags;
170 : int ret;
171 :
172 0 : spin_lock_irqsave(&nh->lock, flags);
173 0 : ret = notifier_chain_register(&nh->head, n, true);
174 0 : spin_unlock_irqrestore(&nh->lock, flags);
175 0 : return ret;
176 : }
177 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
178 :
179 : /**
180 : * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
181 : * @nh: Pointer to head of the atomic notifier chain
182 : * @n: Entry to remove from notifier chain
183 : *
184 : * Removes a notifier from an atomic notifier chain.
185 : *
186 : * Returns zero on success or %-ENOENT on failure.
187 : */
188 0 : int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
189 : struct notifier_block *n)
190 : {
191 : unsigned long flags;
192 : int ret;
193 :
194 0 : spin_lock_irqsave(&nh->lock, flags);
195 0 : ret = notifier_chain_unregister(&nh->head, n);
196 0 : spin_unlock_irqrestore(&nh->lock, flags);
197 0 : synchronize_rcu();
198 0 : return ret;
199 : }
200 : EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
201 :
202 : /**
203 : * atomic_notifier_call_chain - Call functions in an atomic notifier chain
204 : * @nh: Pointer to head of the atomic notifier chain
205 : * @val: Value passed unmodified to notifier function
206 : * @v: Pointer passed unmodified to notifier function
207 : *
208 : * Calls each function in a notifier chain in turn. The functions
209 : * run in an atomic context, so they must not block.
210 : * This routine uses RCU to synchronize with changes to the chain.
211 : *
212 : * If the return value of the notifier can be and'ed
213 : * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
214 : * will return immediately, with the return value of
215 : * the notifier function which halted execution.
216 : * Otherwise the return value is the return value
217 : * of the last notifier function called.
218 : */
219 1 : int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
220 : unsigned long val, void *v)
221 : {
222 : int ret;
223 :
224 : rcu_read_lock();
225 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
226 : rcu_read_unlock();
227 :
228 1 : return ret;
229 : }
230 : EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
231 : NOKPROBE_SYMBOL(atomic_notifier_call_chain);
232 :
233 : /**
234 : * atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
235 : * @nh: Pointer to head of the atomic notifier chain
236 : *
237 : * Checks whether notifier chain is empty.
238 : *
239 : * Returns true is notifier chain is empty, false otherwise.
240 : */
241 0 : bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
242 : {
243 0 : return !rcu_access_pointer(nh->head);
244 : }
245 :
246 : /*
247 : * Blocking notifier chain routines. All access to the chain is
248 : * synchronized by an rwsem.
249 : */
250 :
251 6 : static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
252 : struct notifier_block *n,
253 : bool unique_priority)
254 : {
255 : int ret;
256 :
257 : /*
258 : * This code gets used during boot-up, when task switching is
259 : * not yet working and interrupts must remain disabled. At
260 : * such times we must not call down_write().
261 : */
262 6 : if (unlikely(system_state == SYSTEM_BOOTING))
263 1 : return notifier_chain_register(&nh->head, n, unique_priority);
264 :
265 5 : down_write(&nh->rwsem);
266 5 : ret = notifier_chain_register(&nh->head, n, unique_priority);
267 5 : up_write(&nh->rwsem);
268 5 : return ret;
269 : }
270 :
271 : /**
272 : * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
273 : * @nh: Pointer to head of the blocking notifier chain
274 : * @n: New entry in notifier chain
275 : *
276 : * Adds a notifier to a blocking notifier chain.
277 : * Must be called in process context.
278 : *
279 : * Returns 0 on success, %-EEXIST on error.
280 : */
281 6 : int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
282 : struct notifier_block *n)
283 : {
284 6 : return __blocking_notifier_chain_register(nh, n, false);
285 : }
286 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
287 :
288 : /**
289 : * blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
290 : * @nh: Pointer to head of the blocking notifier chain
291 : * @n: New entry in notifier chain
292 : *
293 : * Adds a notifier to an blocking notifier chain if there is no other
294 : * notifier registered using the same priority.
295 : *
296 : * Returns 0 on success, %-EEXIST or %-EBUSY on error.
297 : */
298 0 : int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
299 : struct notifier_block *n)
300 : {
301 0 : return __blocking_notifier_chain_register(nh, n, true);
302 : }
303 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
304 :
305 : /**
306 : * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
307 : * @nh: Pointer to head of the blocking notifier chain
308 : * @n: Entry to remove from notifier chain
309 : *
310 : * Removes a notifier from a blocking notifier chain.
311 : * Must be called from process context.
312 : *
313 : * Returns zero on success or %-ENOENT on failure.
314 : */
315 0 : int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
316 : struct notifier_block *n)
317 : {
318 : int ret;
319 :
320 : /*
321 : * This code gets used during boot-up, when task switching is
322 : * not yet working and interrupts must remain disabled. At
323 : * such times we must not call down_write().
324 : */
325 0 : if (unlikely(system_state == SYSTEM_BOOTING))
326 0 : return notifier_chain_unregister(&nh->head, n);
327 :
328 0 : down_write(&nh->rwsem);
329 0 : ret = notifier_chain_unregister(&nh->head, n);
330 0 : up_write(&nh->rwsem);
331 0 : return ret;
332 : }
333 : EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
334 :
335 0 : int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
336 : unsigned long val_up, unsigned long val_down, void *v)
337 : {
338 0 : int ret = NOTIFY_DONE;
339 :
340 : /*
341 : * We check the head outside the lock, but if this access is
342 : * racy then it does not matter what the result of the test
343 : * is, we re-check the list after having taken the lock anyway:
344 : */
345 0 : if (rcu_access_pointer(nh->head)) {
346 0 : down_read(&nh->rwsem);
347 0 : ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
348 0 : up_read(&nh->rwsem);
349 : }
350 0 : return ret;
351 : }
352 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
353 :
354 : /**
355 : * blocking_notifier_call_chain - Call functions in a blocking notifier chain
356 : * @nh: Pointer to head of the blocking notifier chain
357 : * @val: Value passed unmodified to notifier function
358 : * @v: Pointer passed unmodified to notifier function
359 : *
360 : * Calls each function in a notifier chain in turn. The functions
361 : * run in a process context, so they are allowed to block.
362 : *
363 : * If the return value of the notifier can be and'ed
364 : * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
365 : * will return immediately, with the return value of
366 : * the notifier function which halted execution.
367 : * Otherwise the return value is the return value
368 : * of the last notifier function called.
369 : */
370 124 : int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
371 : unsigned long val, void *v)
372 : {
373 124 : int ret = NOTIFY_DONE;
374 :
375 : /*
376 : * We check the head outside the lock, but if this access is
377 : * racy then it does not matter what the result of the test
378 : * is, we re-check the list after having taken the lock anyway:
379 : */
380 124 : if (rcu_access_pointer(nh->head)) {
381 1 : down_read(&nh->rwsem);
382 2 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
383 1 : up_read(&nh->rwsem);
384 : }
385 124 : return ret;
386 : }
387 : EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
388 :
389 : /*
390 : * Raw notifier chain routines. There is no protection;
391 : * the caller must provide it. Use at your own risk!
392 : */
393 :
394 : /**
395 : * raw_notifier_chain_register - Add notifier to a raw notifier chain
396 : * @nh: Pointer to head of the raw notifier chain
397 : * @n: New entry in notifier chain
398 : *
399 : * Adds a notifier to a raw notifier chain.
400 : * All locking must be provided by the caller.
401 : *
402 : * Returns 0 on success, %-EEXIST on error.
403 : */
404 0 : int raw_notifier_chain_register(struct raw_notifier_head *nh,
405 : struct notifier_block *n)
406 : {
407 0 : return notifier_chain_register(&nh->head, n, false);
408 : }
409 : EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
410 :
411 : /**
412 : * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
413 : * @nh: Pointer to head of the raw notifier chain
414 : * @n: Entry to remove from notifier chain
415 : *
416 : * Removes a notifier from a raw notifier chain.
417 : * All locking must be provided by the caller.
418 : *
419 : * Returns zero on success or %-ENOENT on failure.
420 : */
421 0 : int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
422 : struct notifier_block *n)
423 : {
424 0 : return notifier_chain_unregister(&nh->head, n);
425 : }
426 : EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
427 :
428 0 : int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
429 : unsigned long val_up, unsigned long val_down, void *v)
430 : {
431 0 : return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
432 : }
433 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
434 :
435 : /**
436 : * raw_notifier_call_chain - Call functions in a raw notifier chain
437 : * @nh: Pointer to head of the raw notifier chain
438 : * @val: Value passed unmodified to notifier function
439 : * @v: Pointer passed unmodified to notifier function
440 : *
441 : * Calls each function in a notifier chain in turn. The functions
442 : * run in an undefined context.
443 : * All locking must be provided by the caller.
444 : *
445 : * If the return value of the notifier can be and'ed
446 : * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
447 : * will return immediately, with the return value of
448 : * the notifier function which halted execution.
449 : * Otherwise the return value is the return value
450 : * of the last notifier function called.
451 : */
452 2749 : int raw_notifier_call_chain(struct raw_notifier_head *nh,
453 : unsigned long val, void *v)
454 : {
455 5498 : return notifier_call_chain(&nh->head, val, v, -1, NULL);
456 : }
457 : EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
458 :
459 : /*
460 : * SRCU notifier chain routines. Registration and unregistration
461 : * use a mutex, and call_chain is synchronized by SRCU (no locks).
462 : */
463 :
464 : /**
465 : * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
466 : * @nh: Pointer to head of the SRCU notifier chain
467 : * @n: New entry in notifier chain
468 : *
469 : * Adds a notifier to an SRCU notifier chain.
470 : * Must be called in process context.
471 : *
472 : * Returns 0 on success, %-EEXIST on error.
473 : */
474 0 : int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
475 : struct notifier_block *n)
476 : {
477 : int ret;
478 :
479 : /*
480 : * This code gets used during boot-up, when task switching is
481 : * not yet working and interrupts must remain disabled. At
482 : * such times we must not call mutex_lock().
483 : */
484 0 : if (unlikely(system_state == SYSTEM_BOOTING))
485 0 : return notifier_chain_register(&nh->head, n, false);
486 :
487 0 : mutex_lock(&nh->mutex);
488 0 : ret = notifier_chain_register(&nh->head, n, false);
489 0 : mutex_unlock(&nh->mutex);
490 0 : return ret;
491 : }
492 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
493 :
494 : /**
495 : * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
496 : * @nh: Pointer to head of the SRCU notifier chain
497 : * @n: Entry to remove from notifier chain
498 : *
499 : * Removes a notifier from an SRCU notifier chain.
500 : * Must be called from process context.
501 : *
502 : * Returns zero on success or %-ENOENT on failure.
503 : */
504 0 : int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
505 : struct notifier_block *n)
506 : {
507 : int ret;
508 :
509 : /*
510 : * This code gets used during boot-up, when task switching is
511 : * not yet working and interrupts must remain disabled. At
512 : * such times we must not call mutex_lock().
513 : */
514 0 : if (unlikely(system_state == SYSTEM_BOOTING))
515 0 : return notifier_chain_unregister(&nh->head, n);
516 :
517 0 : mutex_lock(&nh->mutex);
518 0 : ret = notifier_chain_unregister(&nh->head, n);
519 0 : mutex_unlock(&nh->mutex);
520 0 : synchronize_srcu(&nh->srcu);
521 0 : return ret;
522 : }
523 : EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
524 :
525 : /**
526 : * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
527 : * @nh: Pointer to head of the SRCU notifier chain
528 : * @val: Value passed unmodified to notifier function
529 : * @v: Pointer passed unmodified to notifier function
530 : *
531 : * Calls each function in a notifier chain in turn. The functions
532 : * run in a process context, so they are allowed to block.
533 : *
534 : * If the return value of the notifier can be and'ed
535 : * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
536 : * will return immediately, with the return value of
537 : * the notifier function which halted execution.
538 : * Otherwise the return value is the return value
539 : * of the last notifier function called.
540 : */
541 0 : int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
542 : unsigned long val, void *v)
543 : {
544 : int ret;
545 : int idx;
546 :
547 0 : idx = srcu_read_lock(&nh->srcu);
548 0 : ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
549 0 : srcu_read_unlock(&nh->srcu, idx);
550 0 : return ret;
551 : }
552 : EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
553 :
554 : /**
555 : * srcu_init_notifier_head - Initialize an SRCU notifier head
556 : * @nh: Pointer to head of the srcu notifier chain
557 : *
558 : * Unlike other sorts of notifier heads, SRCU notifier heads require
559 : * dynamic initialization. Be sure to call this routine before
560 : * calling any of the other SRCU notifier routines for this head.
561 : *
562 : * If an SRCU notifier head is deallocated, it must first be cleaned
563 : * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
564 : * per-cpu data (used by the SRCU mechanism) will leak.
565 : */
566 1 : void srcu_init_notifier_head(struct srcu_notifier_head *nh)
567 : {
568 1 : mutex_init(&nh->mutex);
569 1 : if (init_srcu_struct(&nh->srcu) < 0)
570 0 : BUG();
571 1 : nh->head = NULL;
572 1 : }
573 : EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
574 :
575 : static ATOMIC_NOTIFIER_HEAD(die_chain);
576 :
577 0 : int notrace notify_die(enum die_val val, const char *str,
578 : struct pt_regs *regs, long err, int trap, int sig)
579 : {
580 0 : struct die_args args = {
581 : .regs = regs,
582 : .str = str,
583 : .err = err,
584 : .trapnr = trap,
585 : .signr = sig,
586 :
587 : };
588 : RCU_LOCKDEP_WARN(!rcu_is_watching(),
589 : "notify_die called but RCU thinks we're quiescent");
590 0 : return atomic_notifier_call_chain(&die_chain, val, &args);
591 : }
592 : NOKPROBE_SYMBOL(notify_die);
593 :
594 0 : int register_die_notifier(struct notifier_block *nb)
595 : {
596 0 : return atomic_notifier_chain_register(&die_chain, nb);
597 : }
598 : EXPORT_SYMBOL_GPL(register_die_notifier);
599 :
600 0 : int unregister_die_notifier(struct notifier_block *nb)
601 : {
602 0 : return atomic_notifier_chain_unregister(&die_chain, nb);
603 : }
604 : EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
