Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef _LINUX_JUMP_LABEL_H
3 : #define _LINUX_JUMP_LABEL_H
4 :
5 : /*
6 : * Jump label support
7 : *
8 : * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9 : * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
10 : *
11 : * DEPRECATED API:
12 : *
13 : * The use of 'struct static_key' directly, is now DEPRECATED. In addition
14 : * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
15 : *
16 : * struct static_key false = STATIC_KEY_INIT_FALSE;
17 : * struct static_key true = STATIC_KEY_INIT_TRUE;
18 : * static_key_true()
19 : * static_key_false()
20 : *
21 : * The updated API replacements are:
22 : *
23 : * DEFINE_STATIC_KEY_TRUE(key);
24 : * DEFINE_STATIC_KEY_FALSE(key);
25 : * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
26 : * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
27 : * static_branch_likely()
28 : * static_branch_unlikely()
29 : *
30 : * Jump labels provide an interface to generate dynamic branches using
31 : * self-modifying code. Assuming toolchain and architecture support, if we
32 : * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
33 : * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
34 : * (which defaults to false - and the true block is placed out of line).
35 : * Similarly, we can define an initially true key via
36 : * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
37 : * "if (static_branch_unlikely(&key))", in which case we will generate an
38 : * unconditional branch to the out-of-line true branch. Keys that are
39 : * initially true or false can be using in both static_branch_unlikely()
40 : * and static_branch_likely() statements.
41 : *
42 : * At runtime we can change the branch target by setting the key
43 : * to true via a call to static_branch_enable(), or false using
44 : * static_branch_disable(). If the direction of the branch is switched by
45 : * these calls then we run-time modify the branch target via a
46 : * no-op -> jump or jump -> no-op conversion. For example, for an
47 : * initially false key that is used in an "if (static_branch_unlikely(&key))"
48 : * statement, setting the key to true requires us to patch in a jump
49 : * to the out-of-line of true branch.
50 : *
51 : * In addition to static_branch_{enable,disable}, we can also reference count
52 : * the key or branch direction via static_branch_{inc,dec}. Thus,
53 : * static_branch_inc() can be thought of as a 'make more true' and
54 : * static_branch_dec() as a 'make more false'.
55 : *
56 : * Since this relies on modifying code, the branch modifying functions
57 : * must be considered absolute slow paths (machine wide synchronization etc.).
58 : * OTOH, since the affected branches are unconditional, their runtime overhead
59 : * will be absolutely minimal, esp. in the default (off) case where the total
60 : * effect is a single NOP of appropriate size. The on case will patch in a jump
61 : * to the out-of-line block.
62 : *
63 : * When the control is directly exposed to userspace, it is prudent to delay the
64 : * decrement to avoid high frequency code modifications which can (and do)
65 : * cause significant performance degradation. Struct static_key_deferred and
66 : * static_key_slow_dec_deferred() provide for this.
67 : *
68 : * Lacking toolchain and or architecture support, static keys fall back to a
69 : * simple conditional branch.
70 : *
71 : * Additional babbling in: Documentation/staging/static-keys.rst
72 : */
73 :
74 : #ifndef __ASSEMBLY__
75 :
76 : #include <linux/types.h>
77 : #include <linux/compiler.h>
78 :
79 : extern bool static_key_initialized;
80 :
81 : #define STATIC_KEY_CHECK_USE(key) WARN(!static_key_initialized, \
82 : "%s(): static key '%pS' used before call to jump_label_init()", \
83 : __func__, (key))
84 :
85 : struct static_key {
86 : atomic_t enabled;
87 : #ifdef CONFIG_JUMP_LABEL
88 : /*
89 : * Note:
90 : * To make anonymous unions work with old compilers, the static
91 : * initialization of them requires brackets. This creates a dependency
92 : * on the order of the struct with the initializers. If any fields
93 : * are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
94 : * to be modified.
95 : *
96 : * bit 0 => 1 if key is initially true
97 : * 0 if initially false
98 : * bit 1 => 1 if points to struct static_key_mod
99 : * 0 if points to struct jump_entry
100 : */
101 : union {
102 : unsigned long type;
103 : struct jump_entry *entries;
104 : struct static_key_mod *next;
105 : };
106 : #endif /* CONFIG_JUMP_LABEL */
107 : };
108 :
109 : #endif /* __ASSEMBLY__ */
110 :
111 : #ifdef CONFIG_JUMP_LABEL
112 : #include <asm/jump_label.h>
113 :
114 : #ifndef __ASSEMBLY__
115 : #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
116 :
117 : struct jump_entry {
118 : s32 code;
119 : s32 target;
120 : long key; // key may be far away from the core kernel under KASLR
121 : };
122 :
123 : static inline unsigned long jump_entry_code(const struct jump_entry *entry)
124 : {
125 : return (unsigned long)&entry->code + entry->code;
126 : }
127 :
128 : static inline unsigned long jump_entry_target(const struct jump_entry *entry)
129 : {
130 : return (unsigned long)&entry->target + entry->target;
131 : }
132 :
133 : static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
134 : {
135 : long offset = entry->key & ~3L;
136 :
137 : return (struct static_key *)((unsigned long)&entry->key + offset);
138 : }
139 :
140 : #else
141 :
142 : static inline unsigned long jump_entry_code(const struct jump_entry *entry)
143 : {
144 : return entry->code;
145 : }
146 :
147 : static inline unsigned long jump_entry_target(const struct jump_entry *entry)
148 : {
149 : return entry->target;
150 : }
151 :
152 : static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
153 : {
154 : return (struct static_key *)((unsigned long)entry->key & ~3UL);
155 : }
156 :
157 : #endif
158 :
159 : static inline bool jump_entry_is_branch(const struct jump_entry *entry)
160 : {
161 : return (unsigned long)entry->key & 1UL;
162 : }
163 :
164 : static inline bool jump_entry_is_init(const struct jump_entry *entry)
165 : {
166 : return (unsigned long)entry->key & 2UL;
167 : }
168 :
169 : static inline void jump_entry_set_init(struct jump_entry *entry, bool set)
170 : {
171 : if (set)
172 : entry->key |= 2;
173 : else
174 : entry->key &= ~2;
175 : }
176 :
177 : static inline int jump_entry_size(struct jump_entry *entry)
178 : {
179 : #ifdef JUMP_LABEL_NOP_SIZE
180 : return JUMP_LABEL_NOP_SIZE;
181 : #else
182 : return arch_jump_entry_size(entry);
183 : #endif
184 : }
185 :
186 : #endif
187 : #endif
188 :
189 : #ifndef __ASSEMBLY__
190 :
191 : enum jump_label_type {
192 : JUMP_LABEL_NOP = 0,
193 : JUMP_LABEL_JMP,
194 : };
195 :
196 : struct module;
197 :
198 : #ifdef CONFIG_JUMP_LABEL
199 :
200 : #define JUMP_TYPE_FALSE 0UL
201 : #define JUMP_TYPE_TRUE 1UL
202 : #define JUMP_TYPE_LINKED 2UL
203 : #define JUMP_TYPE_MASK 3UL
204 :
205 : static __always_inline bool static_key_false(struct static_key *key)
206 : {
207 : return arch_static_branch(key, false);
208 : }
209 :
210 : static __always_inline bool static_key_true(struct static_key *key)
211 : {
212 : return !arch_static_branch(key, true);
213 : }
214 :
215 : extern struct jump_entry __start___jump_table[];
216 : extern struct jump_entry __stop___jump_table[];
217 :
218 : extern void jump_label_init(void);
219 : extern void jump_label_lock(void);
220 : extern void jump_label_unlock(void);
221 : extern void arch_jump_label_transform(struct jump_entry *entry,
222 : enum jump_label_type type);
223 : extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
224 : enum jump_label_type type);
225 : extern void arch_jump_label_transform_apply(void);
226 : extern int jump_label_text_reserved(void *start, void *end);
227 : extern bool static_key_slow_inc(struct static_key *key);
228 : extern bool static_key_fast_inc_not_disabled(struct static_key *key);
229 : extern void static_key_slow_dec(struct static_key *key);
230 : extern bool static_key_slow_inc_cpuslocked(struct static_key *key);
231 : extern void static_key_slow_dec_cpuslocked(struct static_key *key);
232 : extern int static_key_count(struct static_key *key);
233 : extern void static_key_enable(struct static_key *key);
234 : extern void static_key_disable(struct static_key *key);
235 : extern void static_key_enable_cpuslocked(struct static_key *key);
236 : extern void static_key_disable_cpuslocked(struct static_key *key);
237 : extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
238 :
239 : /*
240 : * We should be using ATOMIC_INIT() for initializing .enabled, but
241 : * the inclusion of atomic.h is problematic for inclusion of jump_label.h
242 : * in 'low-level' headers. Thus, we are initializing .enabled with a
243 : * raw value, but have added a BUILD_BUG_ON() to catch any issues in
244 : * jump_label_init() see: kernel/jump_label.c.
245 : */
246 : #define STATIC_KEY_INIT_TRUE \
247 : { .enabled = { 1 }, \
248 : { .type = JUMP_TYPE_TRUE } }
249 : #define STATIC_KEY_INIT_FALSE \
250 : { .enabled = { 0 }, \
251 : { .type = JUMP_TYPE_FALSE } }
252 :
253 : #else /* !CONFIG_JUMP_LABEL */
254 :
255 : #include <linux/atomic.h>
256 : #include <linux/bug.h>
257 :
258 : static __always_inline int static_key_count(struct static_key *key)
259 : {
260 6107390 : return arch_atomic_read(&key->enabled);
261 : }
262 :
263 : static __always_inline void jump_label_init(void)
264 : {
265 1 : static_key_initialized = true;
266 : }
267 :
268 : static __always_inline bool static_key_false(struct static_key *key)
269 : {
270 : if (unlikely_notrace(static_key_count(key) > 0))
271 : return true;
272 : return false;
273 : }
274 :
275 : static __always_inline bool static_key_true(struct static_key *key)
276 : {
277 : if (likely_notrace(static_key_count(key) > 0))
278 : return true;
279 : return false;
280 : }
281 :
282 2 : static inline bool static_key_fast_inc_not_disabled(struct static_key *key)
283 : {
284 : int v;
285 :
286 2 : STATIC_KEY_CHECK_USE(key);
287 : /*
288 : * Prevent key->enabled getting negative to follow the same semantics
289 : * as for CONFIG_JUMP_LABEL=y, see kernel/jump_label.c comment.
290 : */
291 4 : v = atomic_read(&key->enabled);
292 : do {
293 2 : if (v < 0 || (v + 1) < 0)
294 : return false;
295 4 : } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));
296 : return true;
297 : }
298 : #define static_key_slow_inc(key) static_key_fast_inc_not_disabled(key)
299 :
300 1 : static inline void static_key_slow_dec(struct static_key *key)
301 : {
302 1 : STATIC_KEY_CHECK_USE(key);
303 2 : atomic_dec(&key->enabled);
304 1 : }
305 :
306 : #define static_key_slow_inc_cpuslocked(key) static_key_slow_inc(key)
307 : #define static_key_slow_dec_cpuslocked(key) static_key_slow_dec(key)
308 :
309 : static inline int jump_label_text_reserved(void *start, void *end)
310 : {
311 : return 0;
312 : }
313 :
314 : static inline void jump_label_lock(void) {}
315 : static inline void jump_label_unlock(void) {}
316 :
317 1 : static inline void static_key_enable(struct static_key *key)
318 : {
319 1 : STATIC_KEY_CHECK_USE(key);
320 :
321 2 : if (atomic_read(&key->enabled) != 0) {
322 0 : WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
323 : return;
324 : }
325 1 : atomic_set(&key->enabled, 1);
326 : }
327 :
328 2 : static inline void static_key_disable(struct static_key *key)
329 : {
330 2 : STATIC_KEY_CHECK_USE(key);
331 :
332 4 : if (atomic_read(&key->enabled) != 1) {
333 4 : WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
334 : return;
335 : }
336 0 : atomic_set(&key->enabled, 0);
337 : }
338 :
339 : #define static_key_enable_cpuslocked(k) static_key_enable((k))
340 : #define static_key_disable_cpuslocked(k) static_key_disable((k))
341 :
342 : #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
343 : #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
344 :
345 : #endif /* CONFIG_JUMP_LABEL */
346 :
347 : #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
348 : #define jump_label_enabled static_key_enabled
349 :
350 : /* -------------------------------------------------------------------------- */
351 :
352 : /*
353 : * Two type wrappers around static_key, such that we can use compile time
354 : * type differentiation to emit the right code.
355 : *
356 : * All the below code is macros in order to play type games.
357 : */
358 :
359 : struct static_key_true {
360 : struct static_key key;
361 : };
362 :
363 : struct static_key_false {
364 : struct static_key key;
365 : };
366 :
367 : #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
368 : #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
369 :
370 : #define DEFINE_STATIC_KEY_TRUE(name) \
371 : struct static_key_true name = STATIC_KEY_TRUE_INIT
372 :
373 : #define DEFINE_STATIC_KEY_TRUE_RO(name) \
374 : struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
375 :
376 : #define DECLARE_STATIC_KEY_TRUE(name) \
377 : extern struct static_key_true name
378 :
379 : #define DEFINE_STATIC_KEY_FALSE(name) \
380 : struct static_key_false name = STATIC_KEY_FALSE_INIT
381 :
382 : #define DEFINE_STATIC_KEY_FALSE_RO(name) \
383 : struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
384 :
385 : #define DECLARE_STATIC_KEY_FALSE(name) \
386 : extern struct static_key_false name
387 :
388 : #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
389 : struct static_key_true name[count] = { \
390 : [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
391 : }
392 :
393 : #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
394 : struct static_key_false name[count] = { \
395 : [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
396 : }
397 :
398 : #define _DEFINE_STATIC_KEY_1(name) DEFINE_STATIC_KEY_TRUE(name)
399 : #define _DEFINE_STATIC_KEY_0(name) DEFINE_STATIC_KEY_FALSE(name)
400 : #define DEFINE_STATIC_KEY_MAYBE(cfg, name) \
401 : __PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
402 :
403 : #define _DEFINE_STATIC_KEY_RO_1(name) DEFINE_STATIC_KEY_TRUE_RO(name)
404 : #define _DEFINE_STATIC_KEY_RO_0(name) DEFINE_STATIC_KEY_FALSE_RO(name)
405 : #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name) \
406 : __PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
407 :
408 : #define _DECLARE_STATIC_KEY_1(name) DECLARE_STATIC_KEY_TRUE(name)
409 : #define _DECLARE_STATIC_KEY_0(name) DECLARE_STATIC_KEY_FALSE(name)
410 : #define DECLARE_STATIC_KEY_MAYBE(cfg, name) \
411 : __PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
412 :
413 : extern bool ____wrong_branch_error(void);
414 :
415 : #define static_key_enabled(x) \
416 : ({ \
417 : if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
418 : !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
419 : !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
420 : ____wrong_branch_error(); \
421 : static_key_count((struct static_key *)x) > 0; \
422 : })
423 :
424 : #ifdef CONFIG_JUMP_LABEL
425 :
426 : /*
427 : * Combine the right initial value (type) with the right branch order
428 : * to generate the desired result.
429 : *
430 : *
431 : * type\branch| likely (1) | unlikely (0)
432 : * -----------+-----------------------+------------------
433 : * | |
434 : * true (1) | ... | ...
435 : * | NOP | JMP L
436 : * | <br-stmts> | 1: ...
437 : * | L: ... |
438 : * | |
439 : * | | L: <br-stmts>
440 : * | | jmp 1b
441 : * | |
442 : * -----------+-----------------------+------------------
443 : * | |
444 : * false (0) | ... | ...
445 : * | JMP L | NOP
446 : * | <br-stmts> | 1: ...
447 : * | L: ... |
448 : * | |
449 : * | | L: <br-stmts>
450 : * | | jmp 1b
451 : * | |
452 : * -----------+-----------------------+------------------
453 : *
454 : * The initial value is encoded in the LSB of static_key::entries,
455 : * type: 0 = false, 1 = true.
456 : *
457 : * The branch type is encoded in the LSB of jump_entry::key,
458 : * branch: 0 = unlikely, 1 = likely.
459 : *
460 : * This gives the following logic table:
461 : *
462 : * enabled type branch instuction
463 : * -----------------------------+-----------
464 : * 0 0 0 | NOP
465 : * 0 0 1 | JMP
466 : * 0 1 0 | NOP
467 : * 0 1 1 | JMP
468 : *
469 : * 1 0 0 | JMP
470 : * 1 0 1 | NOP
471 : * 1 1 0 | JMP
472 : * 1 1 1 | NOP
473 : *
474 : * Which gives the following functions:
475 : *
476 : * dynamic: instruction = enabled ^ branch
477 : * static: instruction = type ^ branch
478 : *
479 : * See jump_label_type() / jump_label_init_type().
480 : */
481 :
482 : #define static_branch_likely(x) \
483 : ({ \
484 : bool branch; \
485 : if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
486 : branch = !arch_static_branch(&(x)->key, true); \
487 : else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
488 : branch = !arch_static_branch_jump(&(x)->key, true); \
489 : else \
490 : branch = ____wrong_branch_error(); \
491 : likely_notrace(branch); \
492 : })
493 :
494 : #define static_branch_unlikely(x) \
495 : ({ \
496 : bool branch; \
497 : if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
498 : branch = arch_static_branch_jump(&(x)->key, false); \
499 : else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
500 : branch = arch_static_branch(&(x)->key, false); \
501 : else \
502 : branch = ____wrong_branch_error(); \
503 : unlikely_notrace(branch); \
504 : })
505 :
506 : #else /* !CONFIG_JUMP_LABEL */
507 :
508 : #define static_branch_likely(x) likely_notrace(static_key_enabled(&(x)->key))
509 : #define static_branch_unlikely(x) unlikely_notrace(static_key_enabled(&(x)->key))
510 :
511 : #endif /* CONFIG_JUMP_LABEL */
512 :
513 : #define static_branch_maybe(config, x) \
514 : (IS_ENABLED(config) ? static_branch_likely(x) \
515 : : static_branch_unlikely(x))
516 :
517 : /*
518 : * Advanced usage; refcount, branch is enabled when: count != 0
519 : */
520 :
521 : #define static_branch_inc(x) static_key_slow_inc(&(x)->key)
522 : #define static_branch_dec(x) static_key_slow_dec(&(x)->key)
523 : #define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
524 : #define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
525 :
526 : /*
527 : * Normal usage; boolean enable/disable.
528 : */
529 :
530 : #define static_branch_enable(x) static_key_enable(&(x)->key)
531 : #define static_branch_disable(x) static_key_disable(&(x)->key)
532 : #define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
533 : #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
534 :
535 : #endif /* __ASSEMBLY__ */
536 :
537 : #endif /* _LINUX_JUMP_LABEL_H */
|
