Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : /*
7 : * This file provides wrappers with KASAN instrumentation for atomic operations.
8 : * To use this functionality an arch's atomic.h file needs to define all
9 : * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 : * this file at the end. This file provides atomic_read() that forwards to
11 : * arch_atomic_read() for actual atomic operation.
12 : * Note: if an arch atomic operation is implemented by means of other atomic
13 : * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 : * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 : * double instrumentation.
16 : */
17 : #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 : #define _LINUX_ATOMIC_INSTRUMENTED_H
19 :
20 : #include <linux/build_bug.h>
21 : #include <linux/compiler.h>
22 : #include <linux/instrumented.h>
23 :
24 : static __always_inline int
25 : atomic_read(const atomic_t *v)
26 : {
27 553329 : instrument_atomic_read(v, sizeof(*v));
28 553329 : return arch_atomic_read(v);
29 : }
30 :
31 : static __always_inline int
32 : atomic_read_acquire(const atomic_t *v)
33 : {
34 0 : instrument_atomic_read(v, sizeof(*v));
35 0 : return arch_atomic_read_acquire(v);
36 : }
37 :
38 : static __always_inline void
39 : atomic_set(atomic_t *v, int i)
40 : {
41 863651 : instrument_atomic_write(v, sizeof(*v));
42 863651 : arch_atomic_set(v, i);
43 : }
44 :
45 : static __always_inline void
46 : atomic_set_release(atomic_t *v, int i)
47 : {
48 : kcsan_release();
49 0 : instrument_atomic_write(v, sizeof(*v));
50 0 : arch_atomic_set_release(v, i);
51 : }
52 :
53 : static __always_inline void
54 : atomic_add(int i, atomic_t *v)
55 : {
56 434 : instrument_atomic_read_write(v, sizeof(*v));
57 434 : arch_atomic_add(i, v);
58 : }
59 :
60 : static __always_inline int
61 : atomic_add_return(int i, atomic_t *v)
62 : {
63 : kcsan_mb();
64 0 : instrument_atomic_read_write(v, sizeof(*v));
65 0 : return arch_atomic_add_return(i, v);
66 : }
67 :
68 : static __always_inline int
69 : atomic_add_return_acquire(int i, atomic_t *v)
70 : {
71 : instrument_atomic_read_write(v, sizeof(*v));
72 : return arch_atomic_add_return_acquire(i, v);
73 : }
74 :
75 : static __always_inline int
76 : atomic_add_return_release(int i, atomic_t *v)
77 : {
78 : kcsan_release();
79 : instrument_atomic_read_write(v, sizeof(*v));
80 : return arch_atomic_add_return_release(i, v);
81 : }
82 :
83 : static __always_inline int
84 : atomic_add_return_relaxed(int i, atomic_t *v)
85 : {
86 : instrument_atomic_read_write(v, sizeof(*v));
87 : return arch_atomic_add_return_relaxed(i, v);
88 : }
89 :
90 : static __always_inline int
91 : atomic_fetch_add(int i, atomic_t *v)
92 : {
93 : kcsan_mb();
94 : instrument_atomic_read_write(v, sizeof(*v));
95 : return arch_atomic_fetch_add(i, v);
96 : }
97 :
98 : static __always_inline int
99 : atomic_fetch_add_acquire(int i, atomic_t *v)
100 : {
101 : instrument_atomic_read_write(v, sizeof(*v));
102 : return arch_atomic_fetch_add_acquire(i, v);
103 : }
104 :
105 : static __always_inline int
106 : atomic_fetch_add_release(int i, atomic_t *v)
107 : {
108 : kcsan_release();
109 : instrument_atomic_read_write(v, sizeof(*v));
110 : return arch_atomic_fetch_add_release(i, v);
111 : }
112 :
113 : static __always_inline int
114 : atomic_fetch_add_relaxed(int i, atomic_t *v)
115 : {
116 5198 : instrument_atomic_read_write(v, sizeof(*v));
117 5198 : return arch_atomic_fetch_add_relaxed(i, v);
118 : }
119 :
120 : static __always_inline void
121 : atomic_sub(int i, atomic_t *v)
122 : {
123 8644 : instrument_atomic_read_write(v, sizeof(*v));
124 8644 : arch_atomic_sub(i, v);
125 : }
126 :
127 : static __always_inline int
128 : atomic_sub_return(int i, atomic_t *v)
129 : {
130 : kcsan_mb();
131 51 : instrument_atomic_read_write(v, sizeof(*v));
132 51 : return arch_atomic_sub_return(i, v);
133 : }
134 :
135 : static __always_inline int
136 : atomic_sub_return_acquire(int i, atomic_t *v)
137 : {
138 : instrument_atomic_read_write(v, sizeof(*v));
139 : return arch_atomic_sub_return_acquire(i, v);
140 : }
141 :
142 : static __always_inline int
143 : atomic_sub_return_release(int i, atomic_t *v)
144 : {
145 : kcsan_release();
146 : instrument_atomic_read_write(v, sizeof(*v));
147 : return arch_atomic_sub_return_release(i, v);
148 : }
149 :
150 : static __always_inline int
151 : atomic_sub_return_relaxed(int i, atomic_t *v)
152 : {
153 : instrument_atomic_read_write(v, sizeof(*v));
154 : return arch_atomic_sub_return_relaxed(i, v);
155 : }
156 :
157 : static __always_inline int
158 : atomic_fetch_sub(int i, atomic_t *v)
159 : {
160 : kcsan_mb();
161 : instrument_atomic_read_write(v, sizeof(*v));
162 : return arch_atomic_fetch_sub(i, v);
163 : }
164 :
165 : static __always_inline int
166 : atomic_fetch_sub_acquire(int i, atomic_t *v)
167 : {
168 : instrument_atomic_read_write(v, sizeof(*v));
169 : return arch_atomic_fetch_sub_acquire(i, v);
170 : }
171 :
172 : static __always_inline int
173 : atomic_fetch_sub_release(int i, atomic_t *v)
174 : {
175 : kcsan_release();
176 4880 : instrument_atomic_read_write(v, sizeof(*v));
177 4880 : return arch_atomic_fetch_sub_release(i, v);
178 : }
179 :
180 : static __always_inline int
181 : atomic_fetch_sub_relaxed(int i, atomic_t *v)
182 : {
183 : instrument_atomic_read_write(v, sizeof(*v));
184 : return arch_atomic_fetch_sub_relaxed(i, v);
185 : }
186 :
187 : static __always_inline void
188 : atomic_inc(atomic_t *v)
189 : {
190 17624 : instrument_atomic_read_write(v, sizeof(*v));
191 17624 : arch_atomic_inc(v);
192 : }
193 :
194 : static __always_inline int
195 : atomic_inc_return(atomic_t *v)
196 : {
197 : kcsan_mb();
198 5 : instrument_atomic_read_write(v, sizeof(*v));
199 5 : return arch_atomic_inc_return(v);
200 : }
201 :
202 : static __always_inline int
203 : atomic_inc_return_acquire(atomic_t *v)
204 : {
205 : instrument_atomic_read_write(v, sizeof(*v));
206 : return arch_atomic_inc_return_acquire(v);
207 : }
208 :
209 : static __always_inline int
210 : atomic_inc_return_release(atomic_t *v)
211 : {
212 : kcsan_release();
213 : instrument_atomic_read_write(v, sizeof(*v));
214 : return arch_atomic_inc_return_release(v);
215 : }
216 :
217 : static __always_inline int
218 : atomic_inc_return_relaxed(atomic_t *v)
219 : {
220 0 : instrument_atomic_read_write(v, sizeof(*v));
221 0 : return arch_atomic_inc_return_relaxed(v);
222 : }
223 :
224 : static __always_inline int
225 : atomic_fetch_inc(atomic_t *v)
226 : {
227 : kcsan_mb();
228 0 : instrument_atomic_read_write(v, sizeof(*v));
229 0 : return arch_atomic_fetch_inc(v);
230 : }
231 :
232 : static __always_inline int
233 : atomic_fetch_inc_acquire(atomic_t *v)
234 : {
235 : instrument_atomic_read_write(v, sizeof(*v));
236 : return arch_atomic_fetch_inc_acquire(v);
237 : }
238 :
239 : static __always_inline int
240 : atomic_fetch_inc_release(atomic_t *v)
241 : {
242 : kcsan_release();
243 : instrument_atomic_read_write(v, sizeof(*v));
244 : return arch_atomic_fetch_inc_release(v);
245 : }
246 :
247 : static __always_inline int
248 : atomic_fetch_inc_relaxed(atomic_t *v)
249 : {
250 : instrument_atomic_read_write(v, sizeof(*v));
251 : return arch_atomic_fetch_inc_relaxed(v);
252 : }
253 :
254 : static __always_inline void
255 : atomic_dec(atomic_t *v)
256 : {
257 18 : instrument_atomic_read_write(v, sizeof(*v));
258 18 : arch_atomic_dec(v);
259 : }
260 :
261 : static __always_inline int
262 : atomic_dec_return(atomic_t *v)
263 : {
264 : kcsan_mb();
265 0 : instrument_atomic_read_write(v, sizeof(*v));
266 0 : return arch_atomic_dec_return(v);
267 : }
268 :
269 : static __always_inline int
270 : atomic_dec_return_acquire(atomic_t *v)
271 : {
272 : instrument_atomic_read_write(v, sizeof(*v));
273 : return arch_atomic_dec_return_acquire(v);
274 : }
275 :
276 : static __always_inline int
277 : atomic_dec_return_release(atomic_t *v)
278 : {
279 : kcsan_release();
280 : instrument_atomic_read_write(v, sizeof(*v));
281 : return arch_atomic_dec_return_release(v);
282 : }
283 :
284 : static __always_inline int
285 : atomic_dec_return_relaxed(atomic_t *v)
286 : {
287 0 : instrument_atomic_read_write(v, sizeof(*v));
288 0 : return arch_atomic_dec_return_relaxed(v);
289 : }
290 :
291 : static __always_inline int
292 : atomic_fetch_dec(atomic_t *v)
293 : {
294 : kcsan_mb();
295 : instrument_atomic_read_write(v, sizeof(*v));
296 : return arch_atomic_fetch_dec(v);
297 : }
298 :
299 : static __always_inline int
300 : atomic_fetch_dec_acquire(atomic_t *v)
301 : {
302 : instrument_atomic_read_write(v, sizeof(*v));
303 : return arch_atomic_fetch_dec_acquire(v);
304 : }
305 :
306 : static __always_inline int
307 : atomic_fetch_dec_release(atomic_t *v)
308 : {
309 : kcsan_release();
310 : instrument_atomic_read_write(v, sizeof(*v));
311 : return arch_atomic_fetch_dec_release(v);
312 : }
313 :
314 : static __always_inline int
315 : atomic_fetch_dec_relaxed(atomic_t *v)
316 : {
317 : instrument_atomic_read_write(v, sizeof(*v));
318 : return arch_atomic_fetch_dec_relaxed(v);
319 : }
320 :
321 : static __always_inline void
322 : atomic_and(int i, atomic_t *v)
323 : {
324 : instrument_atomic_read_write(v, sizeof(*v));
325 : arch_atomic_and(i, v);
326 : }
327 :
328 : static __always_inline int
329 : atomic_fetch_and(int i, atomic_t *v)
330 : {
331 : kcsan_mb();
332 : instrument_atomic_read_write(v, sizeof(*v));
333 : return arch_atomic_fetch_and(i, v);
334 : }
335 :
336 : static __always_inline int
337 : atomic_fetch_and_acquire(int i, atomic_t *v)
338 : {
339 : instrument_atomic_read_write(v, sizeof(*v));
340 : return arch_atomic_fetch_and_acquire(i, v);
341 : }
342 :
343 : static __always_inline int
344 : atomic_fetch_and_release(int i, atomic_t *v)
345 : {
346 : kcsan_release();
347 : instrument_atomic_read_write(v, sizeof(*v));
348 : return arch_atomic_fetch_and_release(i, v);
349 : }
350 :
351 : static __always_inline int
352 : atomic_fetch_and_relaxed(int i, atomic_t *v)
353 : {
354 : instrument_atomic_read_write(v, sizeof(*v));
355 : return arch_atomic_fetch_and_relaxed(i, v);
356 : }
357 :
358 : static __always_inline void
359 : atomic_andnot(int i, atomic_t *v)
360 : {
361 0 : instrument_atomic_read_write(v, sizeof(*v));
362 0 : arch_atomic_andnot(i, v);
363 : }
364 :
365 : static __always_inline int
366 : atomic_fetch_andnot(int i, atomic_t *v)
367 : {
368 : kcsan_mb();
369 : instrument_atomic_read_write(v, sizeof(*v));
370 : return arch_atomic_fetch_andnot(i, v);
371 : }
372 :
373 : static __always_inline int
374 : atomic_fetch_andnot_acquire(int i, atomic_t *v)
375 : {
376 : instrument_atomic_read_write(v, sizeof(*v));
377 : return arch_atomic_fetch_andnot_acquire(i, v);
378 : }
379 :
380 : static __always_inline int
381 : atomic_fetch_andnot_release(int i, atomic_t *v)
382 : {
383 : kcsan_release();
384 : instrument_atomic_read_write(v, sizeof(*v));
385 : return arch_atomic_fetch_andnot_release(i, v);
386 : }
387 :
388 : static __always_inline int
389 : atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390 : {
391 : instrument_atomic_read_write(v, sizeof(*v));
392 : return arch_atomic_fetch_andnot_relaxed(i, v);
393 : }
394 :
395 : static __always_inline void
396 : atomic_or(int i, atomic_t *v)
397 : {
398 0 : instrument_atomic_read_write(v, sizeof(*v));
399 0 : arch_atomic_or(i, v);
400 : }
401 :
402 : static __always_inline int
403 : atomic_fetch_or(int i, atomic_t *v)
404 : {
405 : kcsan_mb();
406 0 : instrument_atomic_read_write(v, sizeof(*v));
407 0 : return arch_atomic_fetch_or(i, v);
408 : }
409 :
410 : static __always_inline int
411 : atomic_fetch_or_acquire(int i, atomic_t *v)
412 : {
413 : instrument_atomic_read_write(v, sizeof(*v));
414 : return arch_atomic_fetch_or_acquire(i, v);
415 : }
416 :
417 : static __always_inline int
418 : atomic_fetch_or_release(int i, atomic_t *v)
419 : {
420 : kcsan_release();
421 : instrument_atomic_read_write(v, sizeof(*v));
422 : return arch_atomic_fetch_or_release(i, v);
423 : }
424 :
425 : static __always_inline int
426 : atomic_fetch_or_relaxed(int i, atomic_t *v)
427 : {
428 : instrument_atomic_read_write(v, sizeof(*v));
429 : return arch_atomic_fetch_or_relaxed(i, v);
430 : }
431 :
432 : static __always_inline void
433 : atomic_xor(int i, atomic_t *v)
434 : {
435 : instrument_atomic_read_write(v, sizeof(*v));
436 : arch_atomic_xor(i, v);
437 : }
438 :
439 : static __always_inline int
440 : atomic_fetch_xor(int i, atomic_t *v)
441 : {
442 : kcsan_mb();
443 : instrument_atomic_read_write(v, sizeof(*v));
444 : return arch_atomic_fetch_xor(i, v);
445 : }
446 :
447 : static __always_inline int
448 : atomic_fetch_xor_acquire(int i, atomic_t *v)
449 : {
450 : instrument_atomic_read_write(v, sizeof(*v));
451 : return arch_atomic_fetch_xor_acquire(i, v);
452 : }
453 :
454 : static __always_inline int
455 : atomic_fetch_xor_release(int i, atomic_t *v)
456 : {
457 : kcsan_release();
458 : instrument_atomic_read_write(v, sizeof(*v));
459 : return arch_atomic_fetch_xor_release(i, v);
460 : }
461 :
462 : static __always_inline int
463 : atomic_fetch_xor_relaxed(int i, atomic_t *v)
464 : {
465 : instrument_atomic_read_write(v, sizeof(*v));
466 : return arch_atomic_fetch_xor_relaxed(i, v);
467 : }
468 :
469 : static __always_inline int
470 : atomic_xchg(atomic_t *v, int i)
471 : {
472 : kcsan_mb();
473 0 : instrument_atomic_read_write(v, sizeof(*v));
474 0 : return arch_atomic_xchg(v, i);
475 : }
476 :
477 : static __always_inline int
478 : atomic_xchg_acquire(atomic_t *v, int i)
479 : {
480 : instrument_atomic_read_write(v, sizeof(*v));
481 : return arch_atomic_xchg_acquire(v, i);
482 : }
483 :
484 : static __always_inline int
485 : atomic_xchg_release(atomic_t *v, int i)
486 : {
487 : kcsan_release();
488 : instrument_atomic_read_write(v, sizeof(*v));
489 : return arch_atomic_xchg_release(v, i);
490 : }
491 :
492 : static __always_inline int
493 : atomic_xchg_relaxed(atomic_t *v, int i)
494 : {
495 : instrument_atomic_read_write(v, sizeof(*v));
496 : return arch_atomic_xchg_relaxed(v, i);
497 : }
498 :
499 : static __always_inline int
500 : atomic_cmpxchg(atomic_t *v, int old, int new)
501 : {
502 : kcsan_mb();
503 0 : instrument_atomic_read_write(v, sizeof(*v));
504 0 : return arch_atomic_cmpxchg(v, old, new);
505 : }
506 :
507 : static __always_inline int
508 : atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509 : {
510 : instrument_atomic_read_write(v, sizeof(*v));
511 : return arch_atomic_cmpxchg_acquire(v, old, new);
512 : }
513 :
514 : static __always_inline int
515 : atomic_cmpxchg_release(atomic_t *v, int old, int new)
516 : {
517 : kcsan_release();
518 : instrument_atomic_read_write(v, sizeof(*v));
519 : return arch_atomic_cmpxchg_release(v, old, new);
520 : }
521 :
522 : static __always_inline int
523 : atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524 : {
525 : instrument_atomic_read_write(v, sizeof(*v));
526 : return arch_atomic_cmpxchg_relaxed(v, old, new);
527 : }
528 :
529 : static __always_inline bool
530 : atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531 : {
532 : kcsan_mb();
533 2 : instrument_atomic_read_write(v, sizeof(*v));
534 2 : instrument_atomic_read_write(old, sizeof(*old));
535 2 : return arch_atomic_try_cmpxchg(v, old, new);
536 : }
537 :
538 : static __always_inline bool
539 : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540 : {
541 : instrument_atomic_read_write(v, sizeof(*v));
542 : instrument_atomic_read_write(old, sizeof(*old));
543 : return arch_atomic_try_cmpxchg_acquire(v, old, new);
544 : }
545 :
546 : static __always_inline bool
547 : atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548 : {
549 : kcsan_release();
550 324 : instrument_atomic_read_write(v, sizeof(*v));
551 324 : instrument_atomic_read_write(old, sizeof(*old));
552 324 : return arch_atomic_try_cmpxchg_release(v, old, new);
553 : }
554 :
555 : static __always_inline bool
556 : atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557 : {
558 28 : instrument_atomic_read_write(v, sizeof(*v));
559 28 : instrument_atomic_read_write(old, sizeof(*old));
560 28 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561 : }
562 :
563 : static __always_inline bool
564 : atomic_sub_and_test(int i, atomic_t *v)
565 : {
566 : kcsan_mb();
567 0 : instrument_atomic_read_write(v, sizeof(*v));
568 0 : return arch_atomic_sub_and_test(i, v);
569 : }
570 :
571 : static __always_inline bool
572 : atomic_dec_and_test(atomic_t *v)
573 : {
574 : kcsan_mb();
575 51471 : instrument_atomic_read_write(v, sizeof(*v));
576 51471 : return arch_atomic_dec_and_test(v);
577 : }
578 :
579 : static __always_inline bool
580 : atomic_inc_and_test(atomic_t *v)
581 : {
582 : kcsan_mb();
583 0 : instrument_atomic_read_write(v, sizeof(*v));
584 0 : return arch_atomic_inc_and_test(v);
585 : }
586 :
587 : static __always_inline bool
588 : atomic_add_negative(int i, atomic_t *v)
589 : {
590 : kcsan_mb();
591 340 : instrument_atomic_read_write(v, sizeof(*v));
592 340 : return arch_atomic_add_negative(i, v);
593 : }
594 :
595 : static __always_inline int
596 : atomic_fetch_add_unless(atomic_t *v, int a, int u)
597 : {
598 : kcsan_mb();
599 : instrument_atomic_read_write(v, sizeof(*v));
600 : return arch_atomic_fetch_add_unless(v, a, u);
601 : }
602 :
603 : static __always_inline bool
604 : atomic_add_unless(atomic_t *v, int a, int u)
605 : {
606 : kcsan_mb();
607 955 : instrument_atomic_read_write(v, sizeof(*v));
608 955 : return arch_atomic_add_unless(v, a, u);
609 : }
610 :
611 : static __always_inline bool
612 : atomic_inc_not_zero(atomic_t *v)
613 : {
614 : kcsan_mb();
615 0 : instrument_atomic_read_write(v, sizeof(*v));
616 0 : return arch_atomic_inc_not_zero(v);
617 : }
618 :
619 : static __always_inline bool
620 : atomic_inc_unless_negative(atomic_t *v)
621 : {
622 : kcsan_mb();
623 0 : instrument_atomic_read_write(v, sizeof(*v));
624 0 : return arch_atomic_inc_unless_negative(v);
625 : }
626 :
627 : static __always_inline bool
628 : atomic_dec_unless_positive(atomic_t *v)
629 : {
630 : kcsan_mb();
631 0 : instrument_atomic_read_write(v, sizeof(*v));
632 0 : return arch_atomic_dec_unless_positive(v);
633 : }
634 :
635 : static __always_inline int
636 : atomic_dec_if_positive(atomic_t *v)
637 : {
638 : kcsan_mb();
639 0 : instrument_atomic_read_write(v, sizeof(*v));
640 0 : return arch_atomic_dec_if_positive(v);
641 : }
642 :
643 : static __always_inline s64
644 : atomic64_read(const atomic64_t *v)
645 : {
646 0 : instrument_atomic_read(v, sizeof(*v));
647 0 : return arch_atomic64_read(v);
648 : }
649 :
650 : static __always_inline s64
651 : atomic64_read_acquire(const atomic64_t *v)
652 : {
653 : instrument_atomic_read(v, sizeof(*v));
654 : return arch_atomic64_read_acquire(v);
655 : }
656 :
657 : static __always_inline void
658 : atomic64_set(atomic64_t *v, s64 i)
659 : {
660 51 : instrument_atomic_write(v, sizeof(*v));
661 51 : arch_atomic64_set(v, i);
662 : }
663 :
664 : static __always_inline void
665 : atomic64_set_release(atomic64_t *v, s64 i)
666 : {
667 : kcsan_release();
668 : instrument_atomic_write(v, sizeof(*v));
669 : arch_atomic64_set_release(v, i);
670 : }
671 :
672 : static __always_inline void
673 : atomic64_add(s64 i, atomic64_t *v)
674 : {
675 0 : instrument_atomic_read_write(v, sizeof(*v));
676 0 : arch_atomic64_add(i, v);
677 : }
678 :
679 : static __always_inline s64
680 : atomic64_add_return(s64 i, atomic64_t *v)
681 : {
682 : kcsan_mb();
683 1 : instrument_atomic_read_write(v, sizeof(*v));
684 1 : return arch_atomic64_add_return(i, v);
685 : }
686 :
687 : static __always_inline s64
688 : atomic64_add_return_acquire(s64 i, atomic64_t *v)
689 : {
690 : instrument_atomic_read_write(v, sizeof(*v));
691 : return arch_atomic64_add_return_acquire(i, v);
692 : }
693 :
694 : static __always_inline s64
695 : atomic64_add_return_release(s64 i, atomic64_t *v)
696 : {
697 : kcsan_release();
698 : instrument_atomic_read_write(v, sizeof(*v));
699 : return arch_atomic64_add_return_release(i, v);
700 : }
701 :
702 : static __always_inline s64
703 : atomic64_add_return_relaxed(s64 i, atomic64_t *v)
704 : {
705 : instrument_atomic_read_write(v, sizeof(*v));
706 : return arch_atomic64_add_return_relaxed(i, v);
707 : }
708 :
709 : static __always_inline s64
710 : atomic64_fetch_add(s64 i, atomic64_t *v)
711 : {
712 : kcsan_mb();
713 0 : instrument_atomic_read_write(v, sizeof(*v));
714 0 : return arch_atomic64_fetch_add(i, v);
715 : }
716 :
717 : static __always_inline s64
718 : atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
719 : {
720 : instrument_atomic_read_write(v, sizeof(*v));
721 : return arch_atomic64_fetch_add_acquire(i, v);
722 : }
723 :
724 : static __always_inline s64
725 : atomic64_fetch_add_release(s64 i, atomic64_t *v)
726 : {
727 : kcsan_release();
728 : instrument_atomic_read_write(v, sizeof(*v));
729 : return arch_atomic64_fetch_add_release(i, v);
730 : }
731 :
732 : static __always_inline s64
733 : atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
734 : {
735 : instrument_atomic_read_write(v, sizeof(*v));
736 : return arch_atomic64_fetch_add_relaxed(i, v);
737 : }
738 :
739 : static __always_inline void
740 : atomic64_sub(s64 i, atomic64_t *v)
741 : {
742 0 : instrument_atomic_read_write(v, sizeof(*v));
743 0 : arch_atomic64_sub(i, v);
744 : }
745 :
746 : static __always_inline s64
747 : atomic64_sub_return(s64 i, atomic64_t *v)
748 : {
749 : kcsan_mb();
750 : instrument_atomic_read_write(v, sizeof(*v));
751 : return arch_atomic64_sub_return(i, v);
752 : }
753 :
754 : static __always_inline s64
755 : atomic64_sub_return_acquire(s64 i, atomic64_t *v)
756 : {
757 : instrument_atomic_read_write(v, sizeof(*v));
758 : return arch_atomic64_sub_return_acquire(i, v);
759 : }
760 :
761 : static __always_inline s64
762 : atomic64_sub_return_release(s64 i, atomic64_t *v)
763 : {
764 : kcsan_release();
765 : instrument_atomic_read_write(v, sizeof(*v));
766 : return arch_atomic64_sub_return_release(i, v);
767 : }
768 :
769 : static __always_inline s64
770 : atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
771 : {
772 : instrument_atomic_read_write(v, sizeof(*v));
773 : return arch_atomic64_sub_return_relaxed(i, v);
774 : }
775 :
776 : static __always_inline s64
777 : atomic64_fetch_sub(s64 i, atomic64_t *v)
778 : {
779 : kcsan_mb();
780 : instrument_atomic_read_write(v, sizeof(*v));
781 : return arch_atomic64_fetch_sub(i, v);
782 : }
783 :
784 : static __always_inline s64
785 : atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
786 : {
787 : instrument_atomic_read_write(v, sizeof(*v));
788 : return arch_atomic64_fetch_sub_acquire(i, v);
789 : }
790 :
791 : static __always_inline s64
792 : atomic64_fetch_sub_release(s64 i, atomic64_t *v)
793 : {
794 : kcsan_release();
795 : instrument_atomic_read_write(v, sizeof(*v));
796 : return arch_atomic64_fetch_sub_release(i, v);
797 : }
798 :
799 : static __always_inline s64
800 : atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
801 : {
802 : instrument_atomic_read_write(v, sizeof(*v));
803 : return arch_atomic64_fetch_sub_relaxed(i, v);
804 : }
805 :
806 : static __always_inline void
807 : atomic64_inc(atomic64_t *v)
808 : {
809 : instrument_atomic_read_write(v, sizeof(*v));
810 : arch_atomic64_inc(v);
811 : }
812 :
813 : static __always_inline s64
814 : atomic64_inc_return(atomic64_t *v)
815 : {
816 : kcsan_mb();
817 0 : instrument_atomic_read_write(v, sizeof(*v));
818 0 : return arch_atomic64_inc_return(v);
819 : }
820 :
821 : static __always_inline s64
822 : atomic64_inc_return_acquire(atomic64_t *v)
823 : {
824 : instrument_atomic_read_write(v, sizeof(*v));
825 : return arch_atomic64_inc_return_acquire(v);
826 : }
827 :
828 : static __always_inline s64
829 : atomic64_inc_return_release(atomic64_t *v)
830 : {
831 : kcsan_release();
832 : instrument_atomic_read_write(v, sizeof(*v));
833 : return arch_atomic64_inc_return_release(v);
834 : }
835 :
836 : static __always_inline s64
837 : atomic64_inc_return_relaxed(atomic64_t *v)
838 : {
839 : instrument_atomic_read_write(v, sizeof(*v));
840 : return arch_atomic64_inc_return_relaxed(v);
841 : }
842 :
843 : static __always_inline s64
844 : atomic64_fetch_inc(atomic64_t *v)
845 : {
846 : kcsan_mb();
847 : instrument_atomic_read_write(v, sizeof(*v));
848 : return arch_atomic64_fetch_inc(v);
849 : }
850 :
851 : static __always_inline s64
852 : atomic64_fetch_inc_acquire(atomic64_t *v)
853 : {
854 : instrument_atomic_read_write(v, sizeof(*v));
855 : return arch_atomic64_fetch_inc_acquire(v);
856 : }
857 :
858 : static __always_inline s64
859 : atomic64_fetch_inc_release(atomic64_t *v)
860 : {
861 : kcsan_release();
862 : instrument_atomic_read_write(v, sizeof(*v));
863 : return arch_atomic64_fetch_inc_release(v);
864 : }
865 :
866 : static __always_inline s64
867 : atomic64_fetch_inc_relaxed(atomic64_t *v)
868 : {
869 : instrument_atomic_read_write(v, sizeof(*v));
870 : return arch_atomic64_fetch_inc_relaxed(v);
871 : }
872 :
873 : static __always_inline void
874 : atomic64_dec(atomic64_t *v)
875 : {
876 : instrument_atomic_read_write(v, sizeof(*v));
877 : arch_atomic64_dec(v);
878 : }
879 :
880 : static __always_inline s64
881 : atomic64_dec_return(atomic64_t *v)
882 : {
883 : kcsan_mb();
884 : instrument_atomic_read_write(v, sizeof(*v));
885 : return arch_atomic64_dec_return(v);
886 : }
887 :
888 : static __always_inline s64
889 : atomic64_dec_return_acquire(atomic64_t *v)
890 : {
891 : instrument_atomic_read_write(v, sizeof(*v));
892 : return arch_atomic64_dec_return_acquire(v);
893 : }
894 :
895 : static __always_inline s64
896 : atomic64_dec_return_release(atomic64_t *v)
897 : {
898 : kcsan_release();
899 : instrument_atomic_read_write(v, sizeof(*v));
900 : return arch_atomic64_dec_return_release(v);
901 : }
902 :
903 : static __always_inline s64
904 : atomic64_dec_return_relaxed(atomic64_t *v)
905 : {
906 : instrument_atomic_read_write(v, sizeof(*v));
907 : return arch_atomic64_dec_return_relaxed(v);
908 : }
909 :
910 : static __always_inline s64
911 : atomic64_fetch_dec(atomic64_t *v)
912 : {
913 : kcsan_mb();
914 : instrument_atomic_read_write(v, sizeof(*v));
915 : return arch_atomic64_fetch_dec(v);
916 : }
917 :
918 : static __always_inline s64
919 : atomic64_fetch_dec_acquire(atomic64_t *v)
920 : {
921 : instrument_atomic_read_write(v, sizeof(*v));
922 : return arch_atomic64_fetch_dec_acquire(v);
923 : }
924 :
925 : static __always_inline s64
926 : atomic64_fetch_dec_release(atomic64_t *v)
927 : {
928 : kcsan_release();
929 : instrument_atomic_read_write(v, sizeof(*v));
930 : return arch_atomic64_fetch_dec_release(v);
931 : }
932 :
933 : static __always_inline s64
934 : atomic64_fetch_dec_relaxed(atomic64_t *v)
935 : {
936 : instrument_atomic_read_write(v, sizeof(*v));
937 : return arch_atomic64_fetch_dec_relaxed(v);
938 : }
939 :
940 : static __always_inline void
941 : atomic64_and(s64 i, atomic64_t *v)
942 : {
943 : instrument_atomic_read_write(v, sizeof(*v));
944 : arch_atomic64_and(i, v);
945 : }
946 :
947 : static __always_inline s64
948 : atomic64_fetch_and(s64 i, atomic64_t *v)
949 : {
950 : kcsan_mb();
951 : instrument_atomic_read_write(v, sizeof(*v));
952 : return arch_atomic64_fetch_and(i, v);
953 : }
954 :
955 : static __always_inline s64
956 : atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
957 : {
958 : instrument_atomic_read_write(v, sizeof(*v));
959 : return arch_atomic64_fetch_and_acquire(i, v);
960 : }
961 :
962 : static __always_inline s64
963 : atomic64_fetch_and_release(s64 i, atomic64_t *v)
964 : {
965 : kcsan_release();
966 : instrument_atomic_read_write(v, sizeof(*v));
967 : return arch_atomic64_fetch_and_release(i, v);
968 : }
969 :
970 : static __always_inline s64
971 : atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
972 : {
973 : instrument_atomic_read_write(v, sizeof(*v));
974 : return arch_atomic64_fetch_and_relaxed(i, v);
975 : }
976 :
977 : static __always_inline void
978 : atomic64_andnot(s64 i, atomic64_t *v)
979 : {
980 : instrument_atomic_read_write(v, sizeof(*v));
981 : arch_atomic64_andnot(i, v);
982 : }
983 :
984 : static __always_inline s64
985 : atomic64_fetch_andnot(s64 i, atomic64_t *v)
986 : {
987 : kcsan_mb();
988 : instrument_atomic_read_write(v, sizeof(*v));
989 : return arch_atomic64_fetch_andnot(i, v);
990 : }
991 :
992 : static __always_inline s64
993 : atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
994 : {
995 : instrument_atomic_read_write(v, sizeof(*v));
996 : return arch_atomic64_fetch_andnot_acquire(i, v);
997 : }
998 :
999 : static __always_inline s64
1000 : atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1001 : {
1002 : kcsan_release();
1003 : instrument_atomic_read_write(v, sizeof(*v));
1004 : return arch_atomic64_fetch_andnot_release(i, v);
1005 : }
1006 :
1007 : static __always_inline s64
1008 : atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1009 : {
1010 : instrument_atomic_read_write(v, sizeof(*v));
1011 : return arch_atomic64_fetch_andnot_relaxed(i, v);
1012 : }
1013 :
1014 : static __always_inline void
1015 : atomic64_or(s64 i, atomic64_t *v)
1016 : {
1017 : instrument_atomic_read_write(v, sizeof(*v));
1018 : arch_atomic64_or(i, v);
1019 : }
1020 :
1021 : static __always_inline s64
1022 : atomic64_fetch_or(s64 i, atomic64_t *v)
1023 : {
1024 : kcsan_mb();
1025 : instrument_atomic_read_write(v, sizeof(*v));
1026 : return arch_atomic64_fetch_or(i, v);
1027 : }
1028 :
1029 : static __always_inline s64
1030 : atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1031 : {
1032 : instrument_atomic_read_write(v, sizeof(*v));
1033 : return arch_atomic64_fetch_or_acquire(i, v);
1034 : }
1035 :
1036 : static __always_inline s64
1037 : atomic64_fetch_or_release(s64 i, atomic64_t *v)
1038 : {
1039 : kcsan_release();
1040 : instrument_atomic_read_write(v, sizeof(*v));
1041 : return arch_atomic64_fetch_or_release(i, v);
1042 : }
1043 :
1044 : static __always_inline s64
1045 : atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1046 : {
1047 : instrument_atomic_read_write(v, sizeof(*v));
1048 : return arch_atomic64_fetch_or_relaxed(i, v);
1049 : }
1050 :
1051 : static __always_inline void
1052 : atomic64_xor(s64 i, atomic64_t *v)
1053 : {
1054 : instrument_atomic_read_write(v, sizeof(*v));
1055 : arch_atomic64_xor(i, v);
1056 : }
1057 :
1058 : static __always_inline s64
1059 : atomic64_fetch_xor(s64 i, atomic64_t *v)
1060 : {
1061 : kcsan_mb();
1062 : instrument_atomic_read_write(v, sizeof(*v));
1063 : return arch_atomic64_fetch_xor(i, v);
1064 : }
1065 :
1066 : static __always_inline s64
1067 : atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1068 : {
1069 : instrument_atomic_read_write(v, sizeof(*v));
1070 : return arch_atomic64_fetch_xor_acquire(i, v);
1071 : }
1072 :
1073 : static __always_inline s64
1074 : atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1075 : {
1076 : kcsan_release();
1077 : instrument_atomic_read_write(v, sizeof(*v));
1078 : return arch_atomic64_fetch_xor_release(i, v);
1079 : }
1080 :
1081 : static __always_inline s64
1082 : atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1083 : {
1084 : instrument_atomic_read_write(v, sizeof(*v));
1085 : return arch_atomic64_fetch_xor_relaxed(i, v);
1086 : }
1087 :
1088 : static __always_inline s64
1089 : atomic64_xchg(atomic64_t *v, s64 i)
1090 : {
1091 : kcsan_mb();
1092 : instrument_atomic_read_write(v, sizeof(*v));
1093 : return arch_atomic64_xchg(v, i);
1094 : }
1095 :
1096 : static __always_inline s64
1097 : atomic64_xchg_acquire(atomic64_t *v, s64 i)
1098 : {
1099 : instrument_atomic_read_write(v, sizeof(*v));
1100 : return arch_atomic64_xchg_acquire(v, i);
1101 : }
1102 :
1103 : static __always_inline s64
1104 : atomic64_xchg_release(atomic64_t *v, s64 i)
1105 : {
1106 : kcsan_release();
1107 : instrument_atomic_read_write(v, sizeof(*v));
1108 : return arch_atomic64_xchg_release(v, i);
1109 : }
1110 :
1111 : static __always_inline s64
1112 : atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1113 : {
1114 : instrument_atomic_read_write(v, sizeof(*v));
1115 : return arch_atomic64_xchg_relaxed(v, i);
1116 : }
1117 :
1118 : static __always_inline s64
1119 : atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1120 : {
1121 : kcsan_mb();
1122 : instrument_atomic_read_write(v, sizeof(*v));
1123 : return arch_atomic64_cmpxchg(v, old, new);
1124 : }
1125 :
1126 : static __always_inline s64
1127 : atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1128 : {
1129 : instrument_atomic_read_write(v, sizeof(*v));
1130 : return arch_atomic64_cmpxchg_acquire(v, old, new);
1131 : }
1132 :
1133 : static __always_inline s64
1134 : atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1135 : {
1136 : kcsan_release();
1137 : instrument_atomic_read_write(v, sizeof(*v));
1138 : return arch_atomic64_cmpxchg_release(v, old, new);
1139 : }
1140 :
1141 : static __always_inline s64
1142 : atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1143 : {
1144 0 : instrument_atomic_read_write(v, sizeof(*v));
1145 0 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
1146 : }
1147 :
1148 : static __always_inline bool
1149 : atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1150 : {
1151 : kcsan_mb();
1152 0 : instrument_atomic_read_write(v, sizeof(*v));
1153 0 : instrument_atomic_read_write(old, sizeof(*old));
1154 0 : return arch_atomic64_try_cmpxchg(v, old, new);
1155 : }
1156 :
1157 : static __always_inline bool
1158 : atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1159 : {
1160 : instrument_atomic_read_write(v, sizeof(*v));
1161 : instrument_atomic_read_write(old, sizeof(*old));
1162 : return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1163 : }
1164 :
1165 : static __always_inline bool
1166 : atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1167 : {
1168 : kcsan_release();
1169 : instrument_atomic_read_write(v, sizeof(*v));
1170 : instrument_atomic_read_write(old, sizeof(*old));
1171 : return arch_atomic64_try_cmpxchg_release(v, old, new);
1172 : }
1173 :
1174 : static __always_inline bool
1175 : atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1176 : {
1177 : instrument_atomic_read_write(v, sizeof(*v));
1178 : instrument_atomic_read_write(old, sizeof(*old));
1179 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1180 : }
1181 :
1182 : static __always_inline bool
1183 : atomic64_sub_and_test(s64 i, atomic64_t *v)
1184 : {
1185 : kcsan_mb();
1186 : instrument_atomic_read_write(v, sizeof(*v));
1187 : return arch_atomic64_sub_and_test(i, v);
1188 : }
1189 :
1190 : static __always_inline bool
1191 : atomic64_dec_and_test(atomic64_t *v)
1192 : {
1193 : kcsan_mb();
1194 : instrument_atomic_read_write(v, sizeof(*v));
1195 : return arch_atomic64_dec_and_test(v);
1196 : }
1197 :
1198 : static __always_inline bool
1199 : atomic64_inc_and_test(atomic64_t *v)
1200 : {
1201 : kcsan_mb();
1202 : instrument_atomic_read_write(v, sizeof(*v));
1203 : return arch_atomic64_inc_and_test(v);
1204 : }
1205 :
1206 : static __always_inline bool
1207 : atomic64_add_negative(s64 i, atomic64_t *v)
1208 : {
1209 : kcsan_mb();
1210 : instrument_atomic_read_write(v, sizeof(*v));
1211 : return arch_atomic64_add_negative(i, v);
1212 : }
1213 :
1214 : static __always_inline s64
1215 : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1216 : {
1217 : kcsan_mb();
1218 : instrument_atomic_read_write(v, sizeof(*v));
1219 : return arch_atomic64_fetch_add_unless(v, a, u);
1220 : }
1221 :
1222 : static __always_inline bool
1223 : atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1224 : {
1225 : kcsan_mb();
1226 : instrument_atomic_read_write(v, sizeof(*v));
1227 : return arch_atomic64_add_unless(v, a, u);
1228 : }
1229 :
1230 : static __always_inline bool
1231 : atomic64_inc_not_zero(atomic64_t *v)
1232 : {
1233 : kcsan_mb();
1234 : instrument_atomic_read_write(v, sizeof(*v));
1235 : return arch_atomic64_inc_not_zero(v);
1236 : }
1237 :
1238 : static __always_inline bool
1239 : atomic64_inc_unless_negative(atomic64_t *v)
1240 : {
1241 : kcsan_mb();
1242 : instrument_atomic_read_write(v, sizeof(*v));
1243 : return arch_atomic64_inc_unless_negative(v);
1244 : }
1245 :
1246 : static __always_inline bool
1247 : atomic64_dec_unless_positive(atomic64_t *v)
1248 : {
1249 : kcsan_mb();
1250 : instrument_atomic_read_write(v, sizeof(*v));
1251 : return arch_atomic64_dec_unless_positive(v);
1252 : }
1253 :
1254 : static __always_inline s64
1255 : atomic64_dec_if_positive(atomic64_t *v)
1256 : {
1257 : kcsan_mb();
1258 : instrument_atomic_read_write(v, sizeof(*v));
1259 : return arch_atomic64_dec_if_positive(v);
1260 : }
1261 :
1262 : static __always_inline long
1263 : atomic_long_read(const atomic_long_t *v)
1264 : {
1265 28232 : instrument_atomic_read(v, sizeof(*v));
1266 28232 : return arch_atomic_long_read(v);
1267 : }
1268 :
1269 : static __always_inline long
1270 : atomic_long_read_acquire(const atomic_long_t *v)
1271 : {
1272 : instrument_atomic_read(v, sizeof(*v));
1273 : return arch_atomic_long_read_acquire(v);
1274 : }
1275 :
1276 : static __always_inline void
1277 : atomic_long_set(atomic_long_t *v, long i)
1278 : {
1279 48292 : instrument_atomic_write(v, sizeof(*v));
1280 48292 : arch_atomic_long_set(v, i);
1281 : }
1282 :
1283 : static __always_inline void
1284 : atomic_long_set_release(atomic_long_t *v, long i)
1285 : {
1286 : kcsan_release();
1287 : instrument_atomic_write(v, sizeof(*v));
1288 : arch_atomic_long_set_release(v, i);
1289 : }
1290 :
1291 : static __always_inline void
1292 : atomic_long_add(long i, atomic_long_t *v)
1293 : {
1294 19997 : instrument_atomic_read_write(v, sizeof(*v));
1295 19997 : arch_atomic_long_add(i, v);
1296 : }
1297 :
1298 : static __always_inline long
1299 : atomic_long_add_return(long i, atomic_long_t *v)
1300 : {
1301 : kcsan_mb();
1302 599 : instrument_atomic_read_write(v, sizeof(*v));
1303 599 : return arch_atomic_long_add_return(i, v);
1304 : }
1305 :
1306 : static __always_inline long
1307 : atomic_long_add_return_acquire(long i, atomic_long_t *v)
1308 : {
1309 695 : instrument_atomic_read_write(v, sizeof(*v));
1310 695 : return arch_atomic_long_add_return_acquire(i, v);
1311 : }
1312 :
1313 : static __always_inline long
1314 : atomic_long_add_return_release(long i, atomic_long_t *v)
1315 : {
1316 : kcsan_release();
1317 695 : instrument_atomic_read_write(v, sizeof(*v));
1318 695 : return arch_atomic_long_add_return_release(i, v);
1319 : }
1320 :
1321 : static __always_inline long
1322 : atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1323 : {
1324 : instrument_atomic_read_write(v, sizeof(*v));
1325 : return arch_atomic_long_add_return_relaxed(i, v);
1326 : }
1327 :
1328 : static __always_inline long
1329 : atomic_long_fetch_add(long i, atomic_long_t *v)
1330 : {
1331 : kcsan_mb();
1332 0 : instrument_atomic_read_write(v, sizeof(*v));
1333 0 : return arch_atomic_long_fetch_add(i, v);
1334 : }
1335 :
1336 : static __always_inline long
1337 : atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1338 : {
1339 : instrument_atomic_read_write(v, sizeof(*v));
1340 : return arch_atomic_long_fetch_add_acquire(i, v);
1341 : }
1342 :
1343 : static __always_inline long
1344 : atomic_long_fetch_add_release(long i, atomic_long_t *v)
1345 : {
1346 : kcsan_release();
1347 18230 : instrument_atomic_read_write(v, sizeof(*v));
1348 18230 : return arch_atomic_long_fetch_add_release(i, v);
1349 : }
1350 :
1351 : static __always_inline long
1352 : atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1353 : {
1354 : instrument_atomic_read_write(v, sizeof(*v));
1355 : return arch_atomic_long_fetch_add_relaxed(i, v);
1356 : }
1357 :
1358 : static __always_inline void
1359 : atomic_long_sub(long i, atomic_long_t *v)
1360 : {
1361 2319 : instrument_atomic_read_write(v, sizeof(*v));
1362 2319 : arch_atomic_long_sub(i, v);
1363 : }
1364 :
1365 : static __always_inline long
1366 : atomic_long_sub_return(long i, atomic_long_t *v)
1367 : {
1368 : kcsan_mb();
1369 325 : instrument_atomic_read_write(v, sizeof(*v));
1370 325 : return arch_atomic_long_sub_return(i, v);
1371 : }
1372 :
1373 : static __always_inline long
1374 : atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1375 : {
1376 : instrument_atomic_read_write(v, sizeof(*v));
1377 : return arch_atomic_long_sub_return_acquire(i, v);
1378 : }
1379 :
1380 : static __always_inline long
1381 : atomic_long_sub_return_release(long i, atomic_long_t *v)
1382 : {
1383 : kcsan_release();
1384 : instrument_atomic_read_write(v, sizeof(*v));
1385 : return arch_atomic_long_sub_return_release(i, v);
1386 : }
1387 :
1388 : static __always_inline long
1389 : atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1390 : {
1391 : instrument_atomic_read_write(v, sizeof(*v));
1392 : return arch_atomic_long_sub_return_relaxed(i, v);
1393 : }
1394 :
1395 : static __always_inline long
1396 : atomic_long_fetch_sub(long i, atomic_long_t *v)
1397 : {
1398 : kcsan_mb();
1399 : instrument_atomic_read_write(v, sizeof(*v));
1400 : return arch_atomic_long_fetch_sub(i, v);
1401 : }
1402 :
1403 : static __always_inline long
1404 : atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1405 : {
1406 : instrument_atomic_read_write(v, sizeof(*v));
1407 : return arch_atomic_long_fetch_sub_acquire(i, v);
1408 : }
1409 :
1410 : static __always_inline long
1411 : atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1412 : {
1413 : kcsan_release();
1414 : instrument_atomic_read_write(v, sizeof(*v));
1415 : return arch_atomic_long_fetch_sub_release(i, v);
1416 : }
1417 :
1418 : static __always_inline long
1419 : atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1420 : {
1421 : instrument_atomic_read_write(v, sizeof(*v));
1422 : return arch_atomic_long_fetch_sub_relaxed(i, v);
1423 : }
1424 :
1425 : static __always_inline void
1426 : atomic_long_inc(atomic_long_t *v)
1427 : {
1428 2496 : instrument_atomic_read_write(v, sizeof(*v));
1429 2496 : arch_atomic_long_inc(v);
1430 : }
1431 :
1432 : static __always_inline long
1433 : atomic_long_inc_return(atomic_long_t *v)
1434 : {
1435 : kcsan_mb();
1436 0 : instrument_atomic_read_write(v, sizeof(*v));
1437 0 : return arch_atomic_long_inc_return(v);
1438 : }
1439 :
1440 : static __always_inline long
1441 : atomic_long_inc_return_acquire(atomic_long_t *v)
1442 : {
1443 : instrument_atomic_read_write(v, sizeof(*v));
1444 : return arch_atomic_long_inc_return_acquire(v);
1445 : }
1446 :
1447 : static __always_inline long
1448 : atomic_long_inc_return_release(atomic_long_t *v)
1449 : {
1450 : kcsan_release();
1451 : instrument_atomic_read_write(v, sizeof(*v));
1452 : return arch_atomic_long_inc_return_release(v);
1453 : }
1454 :
1455 : static __always_inline long
1456 : atomic_long_inc_return_relaxed(atomic_long_t *v)
1457 : {
1458 5 : instrument_atomic_read_write(v, sizeof(*v));
1459 5 : return arch_atomic_long_inc_return_relaxed(v);
1460 : }
1461 :
1462 : static __always_inline long
1463 : atomic_long_fetch_inc(atomic_long_t *v)
1464 : {
1465 : kcsan_mb();
1466 : instrument_atomic_read_write(v, sizeof(*v));
1467 : return arch_atomic_long_fetch_inc(v);
1468 : }
1469 :
1470 : static __always_inline long
1471 : atomic_long_fetch_inc_acquire(atomic_long_t *v)
1472 : {
1473 : instrument_atomic_read_write(v, sizeof(*v));
1474 : return arch_atomic_long_fetch_inc_acquire(v);
1475 : }
1476 :
1477 : static __always_inline long
1478 : atomic_long_fetch_inc_release(atomic_long_t *v)
1479 : {
1480 : kcsan_release();
1481 : instrument_atomic_read_write(v, sizeof(*v));
1482 : return arch_atomic_long_fetch_inc_release(v);
1483 : }
1484 :
1485 : static __always_inline long
1486 : atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1487 : {
1488 : instrument_atomic_read_write(v, sizeof(*v));
1489 : return arch_atomic_long_fetch_inc_relaxed(v);
1490 : }
1491 :
1492 : static __always_inline void
1493 : atomic_long_dec(atomic_long_t *v)
1494 : {
1495 2056 : instrument_atomic_read_write(v, sizeof(*v));
1496 2056 : arch_atomic_long_dec(v);
1497 : }
1498 :
1499 : static __always_inline long
1500 : atomic_long_dec_return(atomic_long_t *v)
1501 : {
1502 : kcsan_mb();
1503 : instrument_atomic_read_write(v, sizeof(*v));
1504 : return arch_atomic_long_dec_return(v);
1505 : }
1506 :
1507 : static __always_inline long
1508 : atomic_long_dec_return_acquire(atomic_long_t *v)
1509 : {
1510 : instrument_atomic_read_write(v, sizeof(*v));
1511 : return arch_atomic_long_dec_return_acquire(v);
1512 : }
1513 :
1514 : static __always_inline long
1515 : atomic_long_dec_return_release(atomic_long_t *v)
1516 : {
1517 : kcsan_release();
1518 : instrument_atomic_read_write(v, sizeof(*v));
1519 : return arch_atomic_long_dec_return_release(v);
1520 : }
1521 :
1522 : static __always_inline long
1523 : atomic_long_dec_return_relaxed(atomic_long_t *v)
1524 : {
1525 : instrument_atomic_read_write(v, sizeof(*v));
1526 : return arch_atomic_long_dec_return_relaxed(v);
1527 : }
1528 :
1529 : static __always_inline long
1530 : atomic_long_fetch_dec(atomic_long_t *v)
1531 : {
1532 : kcsan_mb();
1533 : instrument_atomic_read_write(v, sizeof(*v));
1534 : return arch_atomic_long_fetch_dec(v);
1535 : }
1536 :
1537 : static __always_inline long
1538 : atomic_long_fetch_dec_acquire(atomic_long_t *v)
1539 : {
1540 : instrument_atomic_read_write(v, sizeof(*v));
1541 : return arch_atomic_long_fetch_dec_acquire(v);
1542 : }
1543 :
1544 : static __always_inline long
1545 : atomic_long_fetch_dec_release(atomic_long_t *v)
1546 : {
1547 : kcsan_release();
1548 : instrument_atomic_read_write(v, sizeof(*v));
1549 : return arch_atomic_long_fetch_dec_release(v);
1550 : }
1551 :
1552 : static __always_inline long
1553 : atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1554 : {
1555 : instrument_atomic_read_write(v, sizeof(*v));
1556 : return arch_atomic_long_fetch_dec_relaxed(v);
1557 : }
1558 :
1559 : static __always_inline void
1560 : atomic_long_and(long i, atomic_long_t *v)
1561 : {
1562 : instrument_atomic_read_write(v, sizeof(*v));
1563 : arch_atomic_long_and(i, v);
1564 : }
1565 :
1566 : static __always_inline long
1567 : atomic_long_fetch_and(long i, atomic_long_t *v)
1568 : {
1569 : kcsan_mb();
1570 : instrument_atomic_read_write(v, sizeof(*v));
1571 : return arch_atomic_long_fetch_and(i, v);
1572 : }
1573 :
1574 : static __always_inline long
1575 : atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1576 : {
1577 : instrument_atomic_read_write(v, sizeof(*v));
1578 : return arch_atomic_long_fetch_and_acquire(i, v);
1579 : }
1580 :
1581 : static __always_inline long
1582 : atomic_long_fetch_and_release(long i, atomic_long_t *v)
1583 : {
1584 : kcsan_release();
1585 : instrument_atomic_read_write(v, sizeof(*v));
1586 : return arch_atomic_long_fetch_and_release(i, v);
1587 : }
1588 :
1589 : static __always_inline long
1590 : atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1591 : {
1592 : instrument_atomic_read_write(v, sizeof(*v));
1593 : return arch_atomic_long_fetch_and_relaxed(i, v);
1594 : }
1595 :
1596 : static __always_inline void
1597 : atomic_long_andnot(long i, atomic_long_t *v)
1598 : {
1599 0 : instrument_atomic_read_write(v, sizeof(*v));
1600 0 : arch_atomic_long_andnot(i, v);
1601 : }
1602 :
1603 : static __always_inline long
1604 : atomic_long_fetch_andnot(long i, atomic_long_t *v)
1605 : {
1606 : kcsan_mb();
1607 : instrument_atomic_read_write(v, sizeof(*v));
1608 : return arch_atomic_long_fetch_andnot(i, v);
1609 : }
1610 :
1611 : static __always_inline long
1612 : atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1613 : {
1614 : instrument_atomic_read_write(v, sizeof(*v));
1615 : return arch_atomic_long_fetch_andnot_acquire(i, v);
1616 : }
1617 :
1618 : static __always_inline long
1619 : atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1620 : {
1621 : kcsan_release();
1622 : instrument_atomic_read_write(v, sizeof(*v));
1623 : return arch_atomic_long_fetch_andnot_release(i, v);
1624 : }
1625 :
1626 : static __always_inline long
1627 : atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1628 : {
1629 : instrument_atomic_read_write(v, sizeof(*v));
1630 : return arch_atomic_long_fetch_andnot_relaxed(i, v);
1631 : }
1632 :
1633 : static __always_inline void
1634 : atomic_long_or(long i, atomic_long_t *v)
1635 : {
1636 0 : instrument_atomic_read_write(v, sizeof(*v));
1637 0 : arch_atomic_long_or(i, v);
1638 : }
1639 :
1640 : static __always_inline long
1641 : atomic_long_fetch_or(long i, atomic_long_t *v)
1642 : {
1643 : kcsan_mb();
1644 : instrument_atomic_read_write(v, sizeof(*v));
1645 : return arch_atomic_long_fetch_or(i, v);
1646 : }
1647 :
1648 : static __always_inline long
1649 : atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1650 : {
1651 : instrument_atomic_read_write(v, sizeof(*v));
1652 : return arch_atomic_long_fetch_or_acquire(i, v);
1653 : }
1654 :
1655 : static __always_inline long
1656 : atomic_long_fetch_or_release(long i, atomic_long_t *v)
1657 : {
1658 : kcsan_release();
1659 : instrument_atomic_read_write(v, sizeof(*v));
1660 : return arch_atomic_long_fetch_or_release(i, v);
1661 : }
1662 :
1663 : static __always_inline long
1664 : atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1665 : {
1666 : instrument_atomic_read_write(v, sizeof(*v));
1667 : return arch_atomic_long_fetch_or_relaxed(i, v);
1668 : }
1669 :
1670 : static __always_inline void
1671 : atomic_long_xor(long i, atomic_long_t *v)
1672 : {
1673 : instrument_atomic_read_write(v, sizeof(*v));
1674 : arch_atomic_long_xor(i, v);
1675 : }
1676 :
1677 : static __always_inline long
1678 : atomic_long_fetch_xor(long i, atomic_long_t *v)
1679 : {
1680 : kcsan_mb();
1681 : instrument_atomic_read_write(v, sizeof(*v));
1682 : return arch_atomic_long_fetch_xor(i, v);
1683 : }
1684 :
1685 : static __always_inline long
1686 : atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1687 : {
1688 : instrument_atomic_read_write(v, sizeof(*v));
1689 : return arch_atomic_long_fetch_xor_acquire(i, v);
1690 : }
1691 :
1692 : static __always_inline long
1693 : atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1694 : {
1695 : kcsan_release();
1696 : instrument_atomic_read_write(v, sizeof(*v));
1697 : return arch_atomic_long_fetch_xor_release(i, v);
1698 : }
1699 :
1700 : static __always_inline long
1701 : atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1702 : {
1703 : instrument_atomic_read_write(v, sizeof(*v));
1704 : return arch_atomic_long_fetch_xor_relaxed(i, v);
1705 : }
1706 :
1707 : static __always_inline long
1708 : atomic_long_xchg(atomic_long_t *v, long i)
1709 : {
1710 : kcsan_mb();
1711 0 : instrument_atomic_read_write(v, sizeof(*v));
1712 0 : return arch_atomic_long_xchg(v, i);
1713 : }
1714 :
1715 : static __always_inline long
1716 : atomic_long_xchg_acquire(atomic_long_t *v, long i)
1717 : {
1718 : instrument_atomic_read_write(v, sizeof(*v));
1719 : return arch_atomic_long_xchg_acquire(v, i);
1720 : }
1721 :
1722 : static __always_inline long
1723 : atomic_long_xchg_release(atomic_long_t *v, long i)
1724 : {
1725 : kcsan_release();
1726 : instrument_atomic_read_write(v, sizeof(*v));
1727 : return arch_atomic_long_xchg_release(v, i);
1728 : }
1729 :
1730 : static __always_inline long
1731 : atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1732 : {
1733 : instrument_atomic_read_write(v, sizeof(*v));
1734 : return arch_atomic_long_xchg_relaxed(v, i);
1735 : }
1736 :
1737 : static __always_inline long
1738 : atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1739 : {
1740 : kcsan_mb();
1741 1 : instrument_atomic_read_write(v, sizeof(*v));
1742 1 : return arch_atomic_long_cmpxchg(v, old, new);
1743 : }
1744 :
1745 : static __always_inline long
1746 : atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1747 : {
1748 : instrument_atomic_read_write(v, sizeof(*v));
1749 : return arch_atomic_long_cmpxchg_acquire(v, old, new);
1750 : }
1751 :
1752 : static __always_inline long
1753 : atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1754 : {
1755 : kcsan_release();
1756 : instrument_atomic_read_write(v, sizeof(*v));
1757 : return arch_atomic_long_cmpxchg_release(v, old, new);
1758 : }
1759 :
1760 : static __always_inline long
1761 : atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1762 : {
1763 747 : instrument_atomic_read_write(v, sizeof(*v));
1764 747 : return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1765 : }
1766 :
1767 : static __always_inline bool
1768 : atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1769 : {
1770 : kcsan_mb();
1771 3168 : instrument_atomic_read_write(v, sizeof(*v));
1772 3168 : instrument_atomic_read_write(old, sizeof(*old));
1773 3168 : return arch_atomic_long_try_cmpxchg(v, old, new);
1774 : }
1775 :
1776 : static __always_inline bool
1777 : atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1778 : {
1779 23612 : instrument_atomic_read_write(v, sizeof(*v));
1780 23612 : instrument_atomic_read_write(old, sizeof(*old));
1781 23612 : return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1782 : }
1783 :
1784 : static __always_inline bool
1785 : atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1786 : {
1787 : kcsan_release();
1788 5377 : instrument_atomic_read_write(v, sizeof(*v));
1789 5377 : instrument_atomic_read_write(old, sizeof(*old));
1790 5377 : return arch_atomic_long_try_cmpxchg_release(v, old, new);
1791 : }
1792 :
1793 : static __always_inline bool
1794 : atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1795 : {
1796 : instrument_atomic_read_write(v, sizeof(*v));
1797 : instrument_atomic_read_write(old, sizeof(*old));
1798 : return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1799 : }
1800 :
1801 : static __always_inline bool
1802 : atomic_long_sub_and_test(long i, atomic_long_t *v)
1803 : {
1804 : kcsan_mb();
1805 0 : instrument_atomic_read_write(v, sizeof(*v));
1806 0 : return arch_atomic_long_sub_and_test(i, v);
1807 : }
1808 :
1809 : static __always_inline bool
1810 : atomic_long_dec_and_test(atomic_long_t *v)
1811 : {
1812 : kcsan_mb();
1813 0 : instrument_atomic_read_write(v, sizeof(*v));
1814 0 : return arch_atomic_long_dec_and_test(v);
1815 : }
1816 :
1817 : static __always_inline bool
1818 : atomic_long_inc_and_test(atomic_long_t *v)
1819 : {
1820 : kcsan_mb();
1821 : instrument_atomic_read_write(v, sizeof(*v));
1822 : return arch_atomic_long_inc_and_test(v);
1823 : }
1824 :
1825 : static __always_inline bool
1826 : atomic_long_add_negative(long i, atomic_long_t *v)
1827 : {
1828 : kcsan_mb();
1829 : instrument_atomic_read_write(v, sizeof(*v));
1830 : return arch_atomic_long_add_negative(i, v);
1831 : }
1832 :
1833 : static __always_inline long
1834 : atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1835 : {
1836 : kcsan_mb();
1837 : instrument_atomic_read_write(v, sizeof(*v));
1838 : return arch_atomic_long_fetch_add_unless(v, a, u);
1839 : }
1840 :
1841 : static __always_inline bool
1842 : atomic_long_add_unless(atomic_long_t *v, long a, long u)
1843 : {
1844 : kcsan_mb();
1845 0 : instrument_atomic_read_write(v, sizeof(*v));
1846 0 : return arch_atomic_long_add_unless(v, a, u);
1847 : }
1848 :
1849 : static __always_inline bool
1850 : atomic_long_inc_not_zero(atomic_long_t *v)
1851 : {
1852 : kcsan_mb();
1853 0 : instrument_atomic_read_write(v, sizeof(*v));
1854 0 : return arch_atomic_long_inc_not_zero(v);
1855 : }
1856 :
1857 : static __always_inline bool
1858 : atomic_long_inc_unless_negative(atomic_long_t *v)
1859 : {
1860 : kcsan_mb();
1861 : instrument_atomic_read_write(v, sizeof(*v));
1862 : return arch_atomic_long_inc_unless_negative(v);
1863 : }
1864 :
1865 : static __always_inline bool
1866 : atomic_long_dec_unless_positive(atomic_long_t *v)
1867 : {
1868 : kcsan_mb();
1869 : instrument_atomic_read_write(v, sizeof(*v));
1870 : return arch_atomic_long_dec_unless_positive(v);
1871 : }
1872 :
1873 : static __always_inline long
1874 : atomic_long_dec_if_positive(atomic_long_t *v)
1875 : {
1876 : kcsan_mb();
1877 0 : instrument_atomic_read_write(v, sizeof(*v));
1878 0 : return arch_atomic_long_dec_if_positive(v);
1879 : }
1880 :
1881 : #define xchg(ptr, ...) \
1882 : ({ \
1883 : typeof(ptr) __ai_ptr = (ptr); \
1884 : kcsan_mb(); \
1885 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1886 : arch_xchg(__ai_ptr, __VA_ARGS__); \
1887 : })
1888 :
1889 : #define xchg_acquire(ptr, ...) \
1890 : ({ \
1891 : typeof(ptr) __ai_ptr = (ptr); \
1892 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1893 : arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1894 : })
1895 :
1896 : #define xchg_release(ptr, ...) \
1897 : ({ \
1898 : typeof(ptr) __ai_ptr = (ptr); \
1899 : kcsan_release(); \
1900 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1901 : arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1902 : })
1903 :
1904 : #define xchg_relaxed(ptr, ...) \
1905 : ({ \
1906 : typeof(ptr) __ai_ptr = (ptr); \
1907 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1908 : arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1909 : })
1910 :
1911 : #define cmpxchg(ptr, ...) \
1912 : ({ \
1913 : typeof(ptr) __ai_ptr = (ptr); \
1914 : kcsan_mb(); \
1915 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1916 : arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1917 : })
1918 :
1919 : #define cmpxchg_acquire(ptr, ...) \
1920 : ({ \
1921 : typeof(ptr) __ai_ptr = (ptr); \
1922 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1923 : arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1924 : })
1925 :
1926 : #define cmpxchg_release(ptr, ...) \
1927 : ({ \
1928 : typeof(ptr) __ai_ptr = (ptr); \
1929 : kcsan_release(); \
1930 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1931 : arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1932 : })
1933 :
1934 : #define cmpxchg_relaxed(ptr, ...) \
1935 : ({ \
1936 : typeof(ptr) __ai_ptr = (ptr); \
1937 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1938 : arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1939 : })
1940 :
1941 : #define cmpxchg64(ptr, ...) \
1942 : ({ \
1943 : typeof(ptr) __ai_ptr = (ptr); \
1944 : kcsan_mb(); \
1945 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1946 : arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1947 : })
1948 :
1949 : #define cmpxchg64_acquire(ptr, ...) \
1950 : ({ \
1951 : typeof(ptr) __ai_ptr = (ptr); \
1952 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1953 : arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1954 : })
1955 :
1956 : #define cmpxchg64_release(ptr, ...) \
1957 : ({ \
1958 : typeof(ptr) __ai_ptr = (ptr); \
1959 : kcsan_release(); \
1960 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1961 : arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1962 : })
1963 :
1964 : #define cmpxchg64_relaxed(ptr, ...) \
1965 : ({ \
1966 : typeof(ptr) __ai_ptr = (ptr); \
1967 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1968 : arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1969 : })
1970 :
1971 : #define try_cmpxchg(ptr, oldp, ...) \
1972 : ({ \
1973 : typeof(ptr) __ai_ptr = (ptr); \
1974 : typeof(oldp) __ai_oldp = (oldp); \
1975 : kcsan_mb(); \
1976 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1977 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1978 : arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1979 : })
1980 :
1981 : #define try_cmpxchg_acquire(ptr, oldp, ...) \
1982 : ({ \
1983 : typeof(ptr) __ai_ptr = (ptr); \
1984 : typeof(oldp) __ai_oldp = (oldp); \
1985 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1986 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1987 : arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1988 : })
1989 :
1990 : #define try_cmpxchg_release(ptr, oldp, ...) \
1991 : ({ \
1992 : typeof(ptr) __ai_ptr = (ptr); \
1993 : typeof(oldp) __ai_oldp = (oldp); \
1994 : kcsan_release(); \
1995 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1996 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1997 : arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1998 : })
1999 :
2000 : #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2001 : ({ \
2002 : typeof(ptr) __ai_ptr = (ptr); \
2003 : typeof(oldp) __ai_oldp = (oldp); \
2004 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2005 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2006 : arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2007 : })
2008 :
2009 : #define try_cmpxchg64(ptr, oldp, ...) \
2010 : ({ \
2011 : typeof(ptr) __ai_ptr = (ptr); \
2012 : typeof(oldp) __ai_oldp = (oldp); \
2013 : kcsan_mb(); \
2014 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2015 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2016 : arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2017 : })
2018 :
2019 : #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2020 : ({ \
2021 : typeof(ptr) __ai_ptr = (ptr); \
2022 : typeof(oldp) __ai_oldp = (oldp); \
2023 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2024 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2025 : arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2026 : })
2027 :
2028 : #define try_cmpxchg64_release(ptr, oldp, ...) \
2029 : ({ \
2030 : typeof(ptr) __ai_ptr = (ptr); \
2031 : typeof(oldp) __ai_oldp = (oldp); \
2032 : kcsan_release(); \
2033 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2035 : arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2036 : })
2037 :
2038 : #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2039 : ({ \
2040 : typeof(ptr) __ai_ptr = (ptr); \
2041 : typeof(oldp) __ai_oldp = (oldp); \
2042 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 : arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 : })
2046 :
2047 : #define cmpxchg_local(ptr, ...) \
2048 : ({ \
2049 : typeof(ptr) __ai_ptr = (ptr); \
2050 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2051 : arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2052 : })
2053 :
2054 : #define cmpxchg64_local(ptr, ...) \
2055 : ({ \
2056 : typeof(ptr) __ai_ptr = (ptr); \
2057 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2058 : arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2059 : })
2060 :
2061 : #define sync_cmpxchg(ptr, ...) \
2062 : ({ \
2063 : typeof(ptr) __ai_ptr = (ptr); \
2064 : kcsan_mb(); \
2065 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
2066 : arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2067 : })
2068 :
2069 : #define cmpxchg_double(ptr, ...) \
2070 : ({ \
2071 : typeof(ptr) __ai_ptr = (ptr); \
2072 : kcsan_mb(); \
2073 : instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2074 : arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2075 : })
2076 :
2077 :
2078 : #define cmpxchg_double_local(ptr, ...) \
2079 : ({ \
2080 : typeof(ptr) __ai_ptr = (ptr); \
2081 : instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2082 : arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2083 : })
2084 :
2085 : #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2086 : // 764f741eb77a7ad565dc8d99ce2837d5542e8aee
|
