Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : /*
7 : * This file provides wrappers with KASAN instrumentation for atomic operations.
8 : * To use this functionality an arch's atomic.h file needs to define all
9 : * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 : * this file at the end. This file provides atomic_read() that forwards to
11 : * arch_atomic_read() for actual atomic operation.
12 : * Note: if an arch atomic operation is implemented by means of other atomic
13 : * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 : * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 : * double instrumentation.
16 : */
17 : #ifndef _LINUX_ATOMIC_INSTRUMENTED_H
18 : #define _LINUX_ATOMIC_INSTRUMENTED_H
19 :
20 : #include <linux/build_bug.h>
21 : #include <linux/compiler.h>
22 : #include <linux/instrumented.h>
23 :
24 : static __always_inline int
25 : atomic_read(const atomic_t *v)
26 : {
27 38774 : instrument_atomic_read(v, sizeof(*v));
28 38774 : return arch_atomic_read(v);
29 : }
30 :
31 : static __always_inline int
32 : atomic_read_acquire(const atomic_t *v)
33 : {
34 0 : instrument_atomic_read(v, sizeof(*v));
35 0 : return arch_atomic_read_acquire(v);
36 : }
37 :
38 : static __always_inline void
39 : atomic_set(atomic_t *v, int i)
40 : {
41 855484 : instrument_atomic_write(v, sizeof(*v));
42 855484 : arch_atomic_set(v, i);
43 : }
44 :
45 : static __always_inline void
46 : atomic_set_release(atomic_t *v, int i)
47 : {
48 : kcsan_release();
49 0 : instrument_atomic_write(v, sizeof(*v));
50 0 : arch_atomic_set_release(v, i);
51 : }
52 :
53 : static __always_inline void
54 : atomic_add(int i, atomic_t *v)
55 : {
56 524 : instrument_atomic_read_write(v, sizeof(*v));
57 524 : arch_atomic_add(i, v);
58 : }
59 :
60 : static __always_inline int
61 : atomic_add_return(int i, atomic_t *v)
62 : {
63 : kcsan_mb();
64 0 : instrument_atomic_read_write(v, sizeof(*v));
65 0 : return arch_atomic_add_return(i, v);
66 : }
67 :
68 : static __always_inline int
69 : atomic_add_return_acquire(int i, atomic_t *v)
70 : {
71 : instrument_atomic_read_write(v, sizeof(*v));
72 : return arch_atomic_add_return_acquire(i, v);
73 : }
74 :
75 : static __always_inline int
76 : atomic_add_return_release(int i, atomic_t *v)
77 : {
78 : kcsan_release();
79 : instrument_atomic_read_write(v, sizeof(*v));
80 : return arch_atomic_add_return_release(i, v);
81 : }
82 :
83 : static __always_inline int
84 : atomic_add_return_relaxed(int i, atomic_t *v)
85 : {
86 : instrument_atomic_read_write(v, sizeof(*v));
87 : return arch_atomic_add_return_relaxed(i, v);
88 : }
89 :
90 : static __always_inline int
91 : atomic_fetch_add(int i, atomic_t *v)
92 : {
93 : kcsan_mb();
94 : instrument_atomic_read_write(v, sizeof(*v));
95 : return arch_atomic_fetch_add(i, v);
96 : }
97 :
98 : static __always_inline int
99 : atomic_fetch_add_acquire(int i, atomic_t *v)
100 : {
101 : instrument_atomic_read_write(v, sizeof(*v));
102 : return arch_atomic_fetch_add_acquire(i, v);
103 : }
104 :
105 : static __always_inline int
106 : atomic_fetch_add_release(int i, atomic_t *v)
107 : {
108 : kcsan_release();
109 : instrument_atomic_read_write(v, sizeof(*v));
110 : return arch_atomic_fetch_add_release(i, v);
111 : }
112 :
113 : static __always_inline int
114 : atomic_fetch_add_relaxed(int i, atomic_t *v)
115 : {
116 7063 : instrument_atomic_read_write(v, sizeof(*v));
117 7063 : return arch_atomic_fetch_add_relaxed(i, v);
118 : }
119 :
120 : static __always_inline void
121 : atomic_sub(int i, atomic_t *v)
122 : {
123 8760 : instrument_atomic_read_write(v, sizeof(*v));
124 8760 : arch_atomic_sub(i, v);
125 : }
126 :
127 : static __always_inline int
128 : atomic_sub_return(int i, atomic_t *v)
129 : {
130 : kcsan_mb();
131 66 : instrument_atomic_read_write(v, sizeof(*v));
132 66 : return arch_atomic_sub_return(i, v);
133 : }
134 :
135 : static __always_inline int
136 : atomic_sub_return_acquire(int i, atomic_t *v)
137 : {
138 : instrument_atomic_read_write(v, sizeof(*v));
139 : return arch_atomic_sub_return_acquire(i, v);
140 : }
141 :
142 : static __always_inline int
143 : atomic_sub_return_release(int i, atomic_t *v)
144 : {
145 : kcsan_release();
146 : instrument_atomic_read_write(v, sizeof(*v));
147 : return arch_atomic_sub_return_release(i, v);
148 : }
149 :
150 : static __always_inline int
151 : atomic_sub_return_relaxed(int i, atomic_t *v)
152 : {
153 : instrument_atomic_read_write(v, sizeof(*v));
154 : return arch_atomic_sub_return_relaxed(i, v);
155 : }
156 :
157 : static __always_inline int
158 : atomic_fetch_sub(int i, atomic_t *v)
159 : {
160 : kcsan_mb();
161 : instrument_atomic_read_write(v, sizeof(*v));
162 : return arch_atomic_fetch_sub(i, v);
163 : }
164 :
165 : static __always_inline int
166 : atomic_fetch_sub_acquire(int i, atomic_t *v)
167 : {
168 : instrument_atomic_read_write(v, sizeof(*v));
169 : return arch_atomic_fetch_sub_acquire(i, v);
170 : }
171 :
172 : static __always_inline int
173 : atomic_fetch_sub_release(int i, atomic_t *v)
174 : {
175 : kcsan_release();
176 7030 : instrument_atomic_read_write(v, sizeof(*v));
177 7030 : return arch_atomic_fetch_sub_release(i, v);
178 : }
179 :
180 : static __always_inline int
181 : atomic_fetch_sub_relaxed(int i, atomic_t *v)
182 : {
183 : instrument_atomic_read_write(v, sizeof(*v));
184 : return arch_atomic_fetch_sub_relaxed(i, v);
185 : }
186 :
187 : static __always_inline void
188 : atomic_inc(atomic_t *v)
189 : {
190 18235 : instrument_atomic_read_write(v, sizeof(*v));
191 18235 : arch_atomic_inc(v);
192 : }
193 :
194 : static __always_inline int
195 : atomic_inc_return(atomic_t *v)
196 : {
197 : kcsan_mb();
198 5 : instrument_atomic_read_write(v, sizeof(*v));
199 5 : return arch_atomic_inc_return(v);
200 : }
201 :
202 : static __always_inline int
203 : atomic_inc_return_acquire(atomic_t *v)
204 : {
205 : instrument_atomic_read_write(v, sizeof(*v));
206 : return arch_atomic_inc_return_acquire(v);
207 : }
208 :
209 : static __always_inline int
210 : atomic_inc_return_release(atomic_t *v)
211 : {
212 : kcsan_release();
213 : instrument_atomic_read_write(v, sizeof(*v));
214 : return arch_atomic_inc_return_release(v);
215 : }
216 :
217 : static __always_inline int
218 : atomic_inc_return_relaxed(atomic_t *v)
219 : {
220 0 : instrument_atomic_read_write(v, sizeof(*v));
221 0 : return arch_atomic_inc_return_relaxed(v);
222 : }
223 :
224 : static __always_inline int
225 : atomic_fetch_inc(atomic_t *v)
226 : {
227 : kcsan_mb();
228 0 : instrument_atomic_read_write(v, sizeof(*v));
229 0 : return arch_atomic_fetch_inc(v);
230 : }
231 :
232 : static __always_inline int
233 : atomic_fetch_inc_acquire(atomic_t *v)
234 : {
235 : instrument_atomic_read_write(v, sizeof(*v));
236 : return arch_atomic_fetch_inc_acquire(v);
237 : }
238 :
239 : static __always_inline int
240 : atomic_fetch_inc_release(atomic_t *v)
241 : {
242 : kcsan_release();
243 : instrument_atomic_read_write(v, sizeof(*v));
244 : return arch_atomic_fetch_inc_release(v);
245 : }
246 :
247 : static __always_inline int
248 : atomic_fetch_inc_relaxed(atomic_t *v)
249 : {
250 : instrument_atomic_read_write(v, sizeof(*v));
251 : return arch_atomic_fetch_inc_relaxed(v);
252 : }
253 :
254 : static __always_inline void
255 : atomic_dec(atomic_t *v)
256 : {
257 23 : instrument_atomic_read_write(v, sizeof(*v));
258 23 : arch_atomic_dec(v);
259 : }
260 :
261 : static __always_inline int
262 : atomic_dec_return(atomic_t *v)
263 : {
264 : kcsan_mb();
265 0 : instrument_atomic_read_write(v, sizeof(*v));
266 0 : return arch_atomic_dec_return(v);
267 : }
268 :
269 : static __always_inline int
270 : atomic_dec_return_acquire(atomic_t *v)
271 : {
272 : instrument_atomic_read_write(v, sizeof(*v));
273 : return arch_atomic_dec_return_acquire(v);
274 : }
275 :
276 : static __always_inline int
277 : atomic_dec_return_release(atomic_t *v)
278 : {
279 : kcsan_release();
280 : instrument_atomic_read_write(v, sizeof(*v));
281 : return arch_atomic_dec_return_release(v);
282 : }
283 :
284 : static __always_inline int
285 : atomic_dec_return_relaxed(atomic_t *v)
286 : {
287 0 : instrument_atomic_read_write(v, sizeof(*v));
288 0 : return arch_atomic_dec_return_relaxed(v);
289 : }
290 :
291 : static __always_inline int
292 : atomic_fetch_dec(atomic_t *v)
293 : {
294 : kcsan_mb();
295 : instrument_atomic_read_write(v, sizeof(*v));
296 : return arch_atomic_fetch_dec(v);
297 : }
298 :
299 : static __always_inline int
300 : atomic_fetch_dec_acquire(atomic_t *v)
301 : {
302 : instrument_atomic_read_write(v, sizeof(*v));
303 : return arch_atomic_fetch_dec_acquire(v);
304 : }
305 :
306 : static __always_inline int
307 : atomic_fetch_dec_release(atomic_t *v)
308 : {
309 : kcsan_release();
310 : instrument_atomic_read_write(v, sizeof(*v));
311 : return arch_atomic_fetch_dec_release(v);
312 : }
313 :
314 : static __always_inline int
315 : atomic_fetch_dec_relaxed(atomic_t *v)
316 : {
317 : instrument_atomic_read_write(v, sizeof(*v));
318 : return arch_atomic_fetch_dec_relaxed(v);
319 : }
320 :
321 : static __always_inline void
322 : atomic_and(int i, atomic_t *v)
323 : {
324 : instrument_atomic_read_write(v, sizeof(*v));
325 : arch_atomic_and(i, v);
326 : }
327 :
328 : static __always_inline int
329 : atomic_fetch_and(int i, atomic_t *v)
330 : {
331 : kcsan_mb();
332 : instrument_atomic_read_write(v, sizeof(*v));
333 : return arch_atomic_fetch_and(i, v);
334 : }
335 :
336 : static __always_inline int
337 : atomic_fetch_and_acquire(int i, atomic_t *v)
338 : {
339 : instrument_atomic_read_write(v, sizeof(*v));
340 : return arch_atomic_fetch_and_acquire(i, v);
341 : }
342 :
343 : static __always_inline int
344 : atomic_fetch_and_release(int i, atomic_t *v)
345 : {
346 : kcsan_release();
347 : instrument_atomic_read_write(v, sizeof(*v));
348 : return arch_atomic_fetch_and_release(i, v);
349 : }
350 :
351 : static __always_inline int
352 : atomic_fetch_and_relaxed(int i, atomic_t *v)
353 : {
354 : instrument_atomic_read_write(v, sizeof(*v));
355 : return arch_atomic_fetch_and_relaxed(i, v);
356 : }
357 :
358 : static __always_inline void
359 : atomic_andnot(int i, atomic_t *v)
360 : {
361 0 : instrument_atomic_read_write(v, sizeof(*v));
362 0 : arch_atomic_andnot(i, v);
363 : }
364 :
365 : static __always_inline int
366 : atomic_fetch_andnot(int i, atomic_t *v)
367 : {
368 : kcsan_mb();
369 : instrument_atomic_read_write(v, sizeof(*v));
370 : return arch_atomic_fetch_andnot(i, v);
371 : }
372 :
373 : static __always_inline int
374 : atomic_fetch_andnot_acquire(int i, atomic_t *v)
375 : {
376 : instrument_atomic_read_write(v, sizeof(*v));
377 : return arch_atomic_fetch_andnot_acquire(i, v);
378 : }
379 :
380 : static __always_inline int
381 : atomic_fetch_andnot_release(int i, atomic_t *v)
382 : {
383 : kcsan_release();
384 : instrument_atomic_read_write(v, sizeof(*v));
385 : return arch_atomic_fetch_andnot_release(i, v);
386 : }
387 :
388 : static __always_inline int
389 : atomic_fetch_andnot_relaxed(int i, atomic_t *v)
390 : {
391 : instrument_atomic_read_write(v, sizeof(*v));
392 : return arch_atomic_fetch_andnot_relaxed(i, v);
393 : }
394 :
395 : static __always_inline void
396 : atomic_or(int i, atomic_t *v)
397 : {
398 0 : instrument_atomic_read_write(v, sizeof(*v));
399 0 : arch_atomic_or(i, v);
400 : }
401 :
402 : static __always_inline int
403 : atomic_fetch_or(int i, atomic_t *v)
404 : {
405 : kcsan_mb();
406 0 : instrument_atomic_read_write(v, sizeof(*v));
407 0 : return arch_atomic_fetch_or(i, v);
408 : }
409 :
410 : static __always_inline int
411 : atomic_fetch_or_acquire(int i, atomic_t *v)
412 : {
413 : instrument_atomic_read_write(v, sizeof(*v));
414 : return arch_atomic_fetch_or_acquire(i, v);
415 : }
416 :
417 : static __always_inline int
418 : atomic_fetch_or_release(int i, atomic_t *v)
419 : {
420 : kcsan_release();
421 : instrument_atomic_read_write(v, sizeof(*v));
422 : return arch_atomic_fetch_or_release(i, v);
423 : }
424 :
425 : static __always_inline int
426 : atomic_fetch_or_relaxed(int i, atomic_t *v)
427 : {
428 : instrument_atomic_read_write(v, sizeof(*v));
429 : return arch_atomic_fetch_or_relaxed(i, v);
430 : }
431 :
432 : static __always_inline void
433 : atomic_xor(int i, atomic_t *v)
434 : {
435 : instrument_atomic_read_write(v, sizeof(*v));
436 : arch_atomic_xor(i, v);
437 : }
438 :
439 : static __always_inline int
440 : atomic_fetch_xor(int i, atomic_t *v)
441 : {
442 : kcsan_mb();
443 : instrument_atomic_read_write(v, sizeof(*v));
444 : return arch_atomic_fetch_xor(i, v);
445 : }
446 :
447 : static __always_inline int
448 : atomic_fetch_xor_acquire(int i, atomic_t *v)
449 : {
450 : instrument_atomic_read_write(v, sizeof(*v));
451 : return arch_atomic_fetch_xor_acquire(i, v);
452 : }
453 :
454 : static __always_inline int
455 : atomic_fetch_xor_release(int i, atomic_t *v)
456 : {
457 : kcsan_release();
458 : instrument_atomic_read_write(v, sizeof(*v));
459 : return arch_atomic_fetch_xor_release(i, v);
460 : }
461 :
462 : static __always_inline int
463 : atomic_fetch_xor_relaxed(int i, atomic_t *v)
464 : {
465 : instrument_atomic_read_write(v, sizeof(*v));
466 : return arch_atomic_fetch_xor_relaxed(i, v);
467 : }
468 :
469 : static __always_inline int
470 : atomic_xchg(atomic_t *v, int i)
471 : {
472 : kcsan_mb();
473 0 : instrument_atomic_read_write(v, sizeof(*v));
474 0 : return arch_atomic_xchg(v, i);
475 : }
476 :
477 : static __always_inline int
478 : atomic_xchg_acquire(atomic_t *v, int i)
479 : {
480 : instrument_atomic_read_write(v, sizeof(*v));
481 : return arch_atomic_xchg_acquire(v, i);
482 : }
483 :
484 : static __always_inline int
485 : atomic_xchg_release(atomic_t *v, int i)
486 : {
487 : kcsan_release();
488 : instrument_atomic_read_write(v, sizeof(*v));
489 : return arch_atomic_xchg_release(v, i);
490 : }
491 :
492 : static __always_inline int
493 : atomic_xchg_relaxed(atomic_t *v, int i)
494 : {
495 : instrument_atomic_read_write(v, sizeof(*v));
496 : return arch_atomic_xchg_relaxed(v, i);
497 : }
498 :
499 : static __always_inline int
500 : atomic_cmpxchg(atomic_t *v, int old, int new)
501 : {
502 : kcsan_mb();
503 0 : instrument_atomic_read_write(v, sizeof(*v));
504 0 : return arch_atomic_cmpxchg(v, old, new);
505 : }
506 :
507 : static __always_inline int
508 : atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
509 : {
510 : instrument_atomic_read_write(v, sizeof(*v));
511 : return arch_atomic_cmpxchg_acquire(v, old, new);
512 : }
513 :
514 : static __always_inline int
515 : atomic_cmpxchg_release(atomic_t *v, int old, int new)
516 : {
517 : kcsan_release();
518 0 : instrument_atomic_read_write(v, sizeof(*v));
519 0 : return arch_atomic_cmpxchg_release(v, old, new);
520 : }
521 :
522 : static __always_inline int
523 : atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
524 : {
525 : instrument_atomic_read_write(v, sizeof(*v));
526 : return arch_atomic_cmpxchg_relaxed(v, old, new);
527 : }
528 :
529 : static __always_inline bool
530 : atomic_try_cmpxchg(atomic_t *v, int *old, int new)
531 : {
532 : kcsan_mb();
533 2 : instrument_atomic_read_write(v, sizeof(*v));
534 2 : instrument_atomic_read_write(old, sizeof(*old));
535 2 : return arch_atomic_try_cmpxchg(v, old, new);
536 : }
537 :
538 : static __always_inline bool
539 : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
540 : {
541 : instrument_atomic_read_write(v, sizeof(*v));
542 : instrument_atomic_read_write(old, sizeof(*old));
543 : return arch_atomic_try_cmpxchg_acquire(v, old, new);
544 : }
545 :
546 : static __always_inline bool
547 : atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
548 : {
549 : kcsan_release();
550 366 : instrument_atomic_read_write(v, sizeof(*v));
551 366 : instrument_atomic_read_write(old, sizeof(*old));
552 366 : return arch_atomic_try_cmpxchg_release(v, old, new);
553 : }
554 :
555 : static __always_inline bool
556 : atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
557 : {
558 28 : instrument_atomic_read_write(v, sizeof(*v));
559 28 : instrument_atomic_read_write(old, sizeof(*old));
560 28 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
561 : }
562 :
563 : static __always_inline bool
564 : atomic_sub_and_test(int i, atomic_t *v)
565 : {
566 : kcsan_mb();
567 0 : instrument_atomic_read_write(v, sizeof(*v));
568 0 : return arch_atomic_sub_and_test(i, v);
569 : }
570 :
571 : static __always_inline bool
572 : atomic_dec_and_test(atomic_t *v)
573 : {
574 : kcsan_mb();
575 51885 : instrument_atomic_read_write(v, sizeof(*v));
576 51885 : return arch_atomic_dec_and_test(v);
577 : }
578 :
579 : static __always_inline bool
580 : atomic_inc_and_test(atomic_t *v)
581 : {
582 : kcsan_mb();
583 0 : instrument_atomic_read_write(v, sizeof(*v));
584 0 : return arch_atomic_inc_and_test(v);
585 : }
586 :
587 : static __always_inline bool
588 : atomic_add_negative(int i, atomic_t *v)
589 : {
590 : kcsan_mb();
591 382 : instrument_atomic_read_write(v, sizeof(*v));
592 382 : return arch_atomic_add_negative(i, v);
593 : }
594 :
595 : static __always_inline bool
596 : atomic_add_negative_acquire(int i, atomic_t *v)
597 : {
598 : instrument_atomic_read_write(v, sizeof(*v));
599 : return arch_atomic_add_negative_acquire(i, v);
600 : }
601 :
602 : static __always_inline bool
603 : atomic_add_negative_release(int i, atomic_t *v)
604 : {
605 : kcsan_release();
606 : instrument_atomic_read_write(v, sizeof(*v));
607 : return arch_atomic_add_negative_release(i, v);
608 : }
609 :
610 : static __always_inline bool
611 : atomic_add_negative_relaxed(int i, atomic_t *v)
612 : {
613 : instrument_atomic_read_write(v, sizeof(*v));
614 : return arch_atomic_add_negative_relaxed(i, v);
615 : }
616 :
617 : static __always_inline int
618 : atomic_fetch_add_unless(atomic_t *v, int a, int u)
619 : {
620 : kcsan_mb();
621 : instrument_atomic_read_write(v, sizeof(*v));
622 : return arch_atomic_fetch_add_unless(v, a, u);
623 : }
624 :
625 : static __always_inline bool
626 : atomic_add_unless(atomic_t *v, int a, int u)
627 : {
628 : kcsan_mb();
629 1022 : instrument_atomic_read_write(v, sizeof(*v));
630 1022 : return arch_atomic_add_unless(v, a, u);
631 : }
632 :
633 : static __always_inline bool
634 : atomic_inc_not_zero(atomic_t *v)
635 : {
636 : kcsan_mb();
637 0 : instrument_atomic_read_write(v, sizeof(*v));
638 0 : return arch_atomic_inc_not_zero(v);
639 : }
640 :
641 : static __always_inline bool
642 : atomic_inc_unless_negative(atomic_t *v)
643 : {
644 : kcsan_mb();
645 0 : instrument_atomic_read_write(v, sizeof(*v));
646 0 : return arch_atomic_inc_unless_negative(v);
647 : }
648 :
649 : static __always_inline bool
650 : atomic_dec_unless_positive(atomic_t *v)
651 : {
652 : kcsan_mb();
653 0 : instrument_atomic_read_write(v, sizeof(*v));
654 0 : return arch_atomic_dec_unless_positive(v);
655 : }
656 :
657 : static __always_inline int
658 : atomic_dec_if_positive(atomic_t *v)
659 : {
660 : kcsan_mb();
661 0 : instrument_atomic_read_write(v, sizeof(*v));
662 0 : return arch_atomic_dec_if_positive(v);
663 : }
664 :
665 : static __always_inline s64
666 : atomic64_read(const atomic64_t *v)
667 : {
668 0 : instrument_atomic_read(v, sizeof(*v));
669 0 : return arch_atomic64_read(v);
670 : }
671 :
672 : static __always_inline s64
673 : atomic64_read_acquire(const atomic64_t *v)
674 : {
675 : instrument_atomic_read(v, sizeof(*v));
676 : return arch_atomic64_read_acquire(v);
677 : }
678 :
679 : static __always_inline void
680 : atomic64_set(atomic64_t *v, s64 i)
681 : {
682 61 : instrument_atomic_write(v, sizeof(*v));
683 61 : arch_atomic64_set(v, i);
684 : }
685 :
686 : static __always_inline void
687 : atomic64_set_release(atomic64_t *v, s64 i)
688 : {
689 : kcsan_release();
690 : instrument_atomic_write(v, sizeof(*v));
691 : arch_atomic64_set_release(v, i);
692 : }
693 :
694 : static __always_inline void
695 : atomic64_add(s64 i, atomic64_t *v)
696 : {
697 0 : instrument_atomic_read_write(v, sizeof(*v));
698 0 : arch_atomic64_add(i, v);
699 : }
700 :
701 : static __always_inline s64
702 : atomic64_add_return(s64 i, atomic64_t *v)
703 : {
704 : kcsan_mb();
705 1 : instrument_atomic_read_write(v, sizeof(*v));
706 1 : return arch_atomic64_add_return(i, v);
707 : }
708 :
709 : static __always_inline s64
710 : atomic64_add_return_acquire(s64 i, atomic64_t *v)
711 : {
712 : instrument_atomic_read_write(v, sizeof(*v));
713 : return arch_atomic64_add_return_acquire(i, v);
714 : }
715 :
716 : static __always_inline s64
717 : atomic64_add_return_release(s64 i, atomic64_t *v)
718 : {
719 : kcsan_release();
720 : instrument_atomic_read_write(v, sizeof(*v));
721 : return arch_atomic64_add_return_release(i, v);
722 : }
723 :
724 : static __always_inline s64
725 : atomic64_add_return_relaxed(s64 i, atomic64_t *v)
726 : {
727 : instrument_atomic_read_write(v, sizeof(*v));
728 : return arch_atomic64_add_return_relaxed(i, v);
729 : }
730 :
731 : static __always_inline s64
732 : atomic64_fetch_add(s64 i, atomic64_t *v)
733 : {
734 : kcsan_mb();
735 0 : instrument_atomic_read_write(v, sizeof(*v));
736 0 : return arch_atomic64_fetch_add(i, v);
737 : }
738 :
739 : static __always_inline s64
740 : atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
741 : {
742 : instrument_atomic_read_write(v, sizeof(*v));
743 : return arch_atomic64_fetch_add_acquire(i, v);
744 : }
745 :
746 : static __always_inline s64
747 : atomic64_fetch_add_release(s64 i, atomic64_t *v)
748 : {
749 : kcsan_release();
750 : instrument_atomic_read_write(v, sizeof(*v));
751 : return arch_atomic64_fetch_add_release(i, v);
752 : }
753 :
754 : static __always_inline s64
755 : atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
756 : {
757 : instrument_atomic_read_write(v, sizeof(*v));
758 : return arch_atomic64_fetch_add_relaxed(i, v);
759 : }
760 :
761 : static __always_inline void
762 : atomic64_sub(s64 i, atomic64_t *v)
763 : {
764 0 : instrument_atomic_read_write(v, sizeof(*v));
765 0 : arch_atomic64_sub(i, v);
766 : }
767 :
768 : static __always_inline s64
769 : atomic64_sub_return(s64 i, atomic64_t *v)
770 : {
771 : kcsan_mb();
772 : instrument_atomic_read_write(v, sizeof(*v));
773 : return arch_atomic64_sub_return(i, v);
774 : }
775 :
776 : static __always_inline s64
777 : atomic64_sub_return_acquire(s64 i, atomic64_t *v)
778 : {
779 : instrument_atomic_read_write(v, sizeof(*v));
780 : return arch_atomic64_sub_return_acquire(i, v);
781 : }
782 :
783 : static __always_inline s64
784 : atomic64_sub_return_release(s64 i, atomic64_t *v)
785 : {
786 : kcsan_release();
787 : instrument_atomic_read_write(v, sizeof(*v));
788 : return arch_atomic64_sub_return_release(i, v);
789 : }
790 :
791 : static __always_inline s64
792 : atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
793 : {
794 : instrument_atomic_read_write(v, sizeof(*v));
795 : return arch_atomic64_sub_return_relaxed(i, v);
796 : }
797 :
798 : static __always_inline s64
799 : atomic64_fetch_sub(s64 i, atomic64_t *v)
800 : {
801 : kcsan_mb();
802 : instrument_atomic_read_write(v, sizeof(*v));
803 : return arch_atomic64_fetch_sub(i, v);
804 : }
805 :
806 : static __always_inline s64
807 : atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
808 : {
809 : instrument_atomic_read_write(v, sizeof(*v));
810 : return arch_atomic64_fetch_sub_acquire(i, v);
811 : }
812 :
813 : static __always_inline s64
814 : atomic64_fetch_sub_release(s64 i, atomic64_t *v)
815 : {
816 : kcsan_release();
817 : instrument_atomic_read_write(v, sizeof(*v));
818 : return arch_atomic64_fetch_sub_release(i, v);
819 : }
820 :
821 : static __always_inline s64
822 : atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
823 : {
824 : instrument_atomic_read_write(v, sizeof(*v));
825 : return arch_atomic64_fetch_sub_relaxed(i, v);
826 : }
827 :
828 : static __always_inline void
829 : atomic64_inc(atomic64_t *v)
830 : {
831 : instrument_atomic_read_write(v, sizeof(*v));
832 : arch_atomic64_inc(v);
833 : }
834 :
835 : static __always_inline s64
836 : atomic64_inc_return(atomic64_t *v)
837 : {
838 : kcsan_mb();
839 0 : instrument_atomic_read_write(v, sizeof(*v));
840 0 : return arch_atomic64_inc_return(v);
841 : }
842 :
843 : static __always_inline s64
844 : atomic64_inc_return_acquire(atomic64_t *v)
845 : {
846 : instrument_atomic_read_write(v, sizeof(*v));
847 : return arch_atomic64_inc_return_acquire(v);
848 : }
849 :
850 : static __always_inline s64
851 : atomic64_inc_return_release(atomic64_t *v)
852 : {
853 : kcsan_release();
854 : instrument_atomic_read_write(v, sizeof(*v));
855 : return arch_atomic64_inc_return_release(v);
856 : }
857 :
858 : static __always_inline s64
859 : atomic64_inc_return_relaxed(atomic64_t *v)
860 : {
861 : instrument_atomic_read_write(v, sizeof(*v));
862 : return arch_atomic64_inc_return_relaxed(v);
863 : }
864 :
865 : static __always_inline s64
866 : atomic64_fetch_inc(atomic64_t *v)
867 : {
868 : kcsan_mb();
869 : instrument_atomic_read_write(v, sizeof(*v));
870 : return arch_atomic64_fetch_inc(v);
871 : }
872 :
873 : static __always_inline s64
874 : atomic64_fetch_inc_acquire(atomic64_t *v)
875 : {
876 : instrument_atomic_read_write(v, sizeof(*v));
877 : return arch_atomic64_fetch_inc_acquire(v);
878 : }
879 :
880 : static __always_inline s64
881 : atomic64_fetch_inc_release(atomic64_t *v)
882 : {
883 : kcsan_release();
884 : instrument_atomic_read_write(v, sizeof(*v));
885 : return arch_atomic64_fetch_inc_release(v);
886 : }
887 :
888 : static __always_inline s64
889 : atomic64_fetch_inc_relaxed(atomic64_t *v)
890 : {
891 : instrument_atomic_read_write(v, sizeof(*v));
892 : return arch_atomic64_fetch_inc_relaxed(v);
893 : }
894 :
895 : static __always_inline void
896 : atomic64_dec(atomic64_t *v)
897 : {
898 : instrument_atomic_read_write(v, sizeof(*v));
899 : arch_atomic64_dec(v);
900 : }
901 :
902 : static __always_inline s64
903 : atomic64_dec_return(atomic64_t *v)
904 : {
905 : kcsan_mb();
906 : instrument_atomic_read_write(v, sizeof(*v));
907 : return arch_atomic64_dec_return(v);
908 : }
909 :
910 : static __always_inline s64
911 : atomic64_dec_return_acquire(atomic64_t *v)
912 : {
913 : instrument_atomic_read_write(v, sizeof(*v));
914 : return arch_atomic64_dec_return_acquire(v);
915 : }
916 :
917 : static __always_inline s64
918 : atomic64_dec_return_release(atomic64_t *v)
919 : {
920 : kcsan_release();
921 : instrument_atomic_read_write(v, sizeof(*v));
922 : return arch_atomic64_dec_return_release(v);
923 : }
924 :
925 : static __always_inline s64
926 : atomic64_dec_return_relaxed(atomic64_t *v)
927 : {
928 : instrument_atomic_read_write(v, sizeof(*v));
929 : return arch_atomic64_dec_return_relaxed(v);
930 : }
931 :
932 : static __always_inline s64
933 : atomic64_fetch_dec(atomic64_t *v)
934 : {
935 : kcsan_mb();
936 : instrument_atomic_read_write(v, sizeof(*v));
937 : return arch_atomic64_fetch_dec(v);
938 : }
939 :
940 : static __always_inline s64
941 : atomic64_fetch_dec_acquire(atomic64_t *v)
942 : {
943 : instrument_atomic_read_write(v, sizeof(*v));
944 : return arch_atomic64_fetch_dec_acquire(v);
945 : }
946 :
947 : static __always_inline s64
948 : atomic64_fetch_dec_release(atomic64_t *v)
949 : {
950 : kcsan_release();
951 : instrument_atomic_read_write(v, sizeof(*v));
952 : return arch_atomic64_fetch_dec_release(v);
953 : }
954 :
955 : static __always_inline s64
956 : atomic64_fetch_dec_relaxed(atomic64_t *v)
957 : {
958 : instrument_atomic_read_write(v, sizeof(*v));
959 : return arch_atomic64_fetch_dec_relaxed(v);
960 : }
961 :
962 : static __always_inline void
963 : atomic64_and(s64 i, atomic64_t *v)
964 : {
965 : instrument_atomic_read_write(v, sizeof(*v));
966 : arch_atomic64_and(i, v);
967 : }
968 :
969 : static __always_inline s64
970 : atomic64_fetch_and(s64 i, atomic64_t *v)
971 : {
972 : kcsan_mb();
973 : instrument_atomic_read_write(v, sizeof(*v));
974 : return arch_atomic64_fetch_and(i, v);
975 : }
976 :
977 : static __always_inline s64
978 : atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
979 : {
980 : instrument_atomic_read_write(v, sizeof(*v));
981 : return arch_atomic64_fetch_and_acquire(i, v);
982 : }
983 :
984 : static __always_inline s64
985 : atomic64_fetch_and_release(s64 i, atomic64_t *v)
986 : {
987 : kcsan_release();
988 : instrument_atomic_read_write(v, sizeof(*v));
989 : return arch_atomic64_fetch_and_release(i, v);
990 : }
991 :
992 : static __always_inline s64
993 : atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
994 : {
995 : instrument_atomic_read_write(v, sizeof(*v));
996 : return arch_atomic64_fetch_and_relaxed(i, v);
997 : }
998 :
999 : static __always_inline void
1000 : atomic64_andnot(s64 i, atomic64_t *v)
1001 : {
1002 : instrument_atomic_read_write(v, sizeof(*v));
1003 : arch_atomic64_andnot(i, v);
1004 : }
1005 :
1006 : static __always_inline s64
1007 : atomic64_fetch_andnot(s64 i, atomic64_t *v)
1008 : {
1009 : kcsan_mb();
1010 : instrument_atomic_read_write(v, sizeof(*v));
1011 : return arch_atomic64_fetch_andnot(i, v);
1012 : }
1013 :
1014 : static __always_inline s64
1015 : atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1016 : {
1017 : instrument_atomic_read_write(v, sizeof(*v));
1018 : return arch_atomic64_fetch_andnot_acquire(i, v);
1019 : }
1020 :
1021 : static __always_inline s64
1022 : atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1023 : {
1024 : kcsan_release();
1025 : instrument_atomic_read_write(v, sizeof(*v));
1026 : return arch_atomic64_fetch_andnot_release(i, v);
1027 : }
1028 :
1029 : static __always_inline s64
1030 : atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1031 : {
1032 : instrument_atomic_read_write(v, sizeof(*v));
1033 : return arch_atomic64_fetch_andnot_relaxed(i, v);
1034 : }
1035 :
1036 : static __always_inline void
1037 : atomic64_or(s64 i, atomic64_t *v)
1038 : {
1039 : instrument_atomic_read_write(v, sizeof(*v));
1040 : arch_atomic64_or(i, v);
1041 : }
1042 :
1043 : static __always_inline s64
1044 : atomic64_fetch_or(s64 i, atomic64_t *v)
1045 : {
1046 : kcsan_mb();
1047 : instrument_atomic_read_write(v, sizeof(*v));
1048 : return arch_atomic64_fetch_or(i, v);
1049 : }
1050 :
1051 : static __always_inline s64
1052 : atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1053 : {
1054 : instrument_atomic_read_write(v, sizeof(*v));
1055 : return arch_atomic64_fetch_or_acquire(i, v);
1056 : }
1057 :
1058 : static __always_inline s64
1059 : atomic64_fetch_or_release(s64 i, atomic64_t *v)
1060 : {
1061 : kcsan_release();
1062 : instrument_atomic_read_write(v, sizeof(*v));
1063 : return arch_atomic64_fetch_or_release(i, v);
1064 : }
1065 :
1066 : static __always_inline s64
1067 : atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1068 : {
1069 : instrument_atomic_read_write(v, sizeof(*v));
1070 : return arch_atomic64_fetch_or_relaxed(i, v);
1071 : }
1072 :
1073 : static __always_inline void
1074 : atomic64_xor(s64 i, atomic64_t *v)
1075 : {
1076 : instrument_atomic_read_write(v, sizeof(*v));
1077 : arch_atomic64_xor(i, v);
1078 : }
1079 :
1080 : static __always_inline s64
1081 : atomic64_fetch_xor(s64 i, atomic64_t *v)
1082 : {
1083 : kcsan_mb();
1084 : instrument_atomic_read_write(v, sizeof(*v));
1085 : return arch_atomic64_fetch_xor(i, v);
1086 : }
1087 :
1088 : static __always_inline s64
1089 : atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1090 : {
1091 : instrument_atomic_read_write(v, sizeof(*v));
1092 : return arch_atomic64_fetch_xor_acquire(i, v);
1093 : }
1094 :
1095 : static __always_inline s64
1096 : atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1097 : {
1098 : kcsan_release();
1099 : instrument_atomic_read_write(v, sizeof(*v));
1100 : return arch_atomic64_fetch_xor_release(i, v);
1101 : }
1102 :
1103 : static __always_inline s64
1104 : atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1105 : {
1106 : instrument_atomic_read_write(v, sizeof(*v));
1107 : return arch_atomic64_fetch_xor_relaxed(i, v);
1108 : }
1109 :
1110 : static __always_inline s64
1111 : atomic64_xchg(atomic64_t *v, s64 i)
1112 : {
1113 : kcsan_mb();
1114 : instrument_atomic_read_write(v, sizeof(*v));
1115 : return arch_atomic64_xchg(v, i);
1116 : }
1117 :
1118 : static __always_inline s64
1119 : atomic64_xchg_acquire(atomic64_t *v, s64 i)
1120 : {
1121 : instrument_atomic_read_write(v, sizeof(*v));
1122 : return arch_atomic64_xchg_acquire(v, i);
1123 : }
1124 :
1125 : static __always_inline s64
1126 : atomic64_xchg_release(atomic64_t *v, s64 i)
1127 : {
1128 : kcsan_release();
1129 : instrument_atomic_read_write(v, sizeof(*v));
1130 : return arch_atomic64_xchg_release(v, i);
1131 : }
1132 :
1133 : static __always_inline s64
1134 : atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1135 : {
1136 : instrument_atomic_read_write(v, sizeof(*v));
1137 : return arch_atomic64_xchg_relaxed(v, i);
1138 : }
1139 :
1140 : static __always_inline s64
1141 : atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1142 : {
1143 : kcsan_mb();
1144 : instrument_atomic_read_write(v, sizeof(*v));
1145 : return arch_atomic64_cmpxchg(v, old, new);
1146 : }
1147 :
1148 : static __always_inline s64
1149 : atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1150 : {
1151 : instrument_atomic_read_write(v, sizeof(*v));
1152 : return arch_atomic64_cmpxchg_acquire(v, old, new);
1153 : }
1154 :
1155 : static __always_inline s64
1156 : atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1157 : {
1158 : kcsan_release();
1159 : instrument_atomic_read_write(v, sizeof(*v));
1160 : return arch_atomic64_cmpxchg_release(v, old, new);
1161 : }
1162 :
1163 : static __always_inline s64
1164 : atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1165 : {
1166 0 : instrument_atomic_read_write(v, sizeof(*v));
1167 0 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
1168 : }
1169 :
1170 : static __always_inline bool
1171 : atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1172 : {
1173 : kcsan_mb();
1174 0 : instrument_atomic_read_write(v, sizeof(*v));
1175 0 : instrument_atomic_read_write(old, sizeof(*old));
1176 0 : return arch_atomic64_try_cmpxchg(v, old, new);
1177 : }
1178 :
1179 : static __always_inline bool
1180 : atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1181 : {
1182 : instrument_atomic_read_write(v, sizeof(*v));
1183 : instrument_atomic_read_write(old, sizeof(*old));
1184 : return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1185 : }
1186 :
1187 : static __always_inline bool
1188 : atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1189 : {
1190 : kcsan_release();
1191 : instrument_atomic_read_write(v, sizeof(*v));
1192 : instrument_atomic_read_write(old, sizeof(*old));
1193 : return arch_atomic64_try_cmpxchg_release(v, old, new);
1194 : }
1195 :
1196 : static __always_inline bool
1197 : atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1198 : {
1199 : instrument_atomic_read_write(v, sizeof(*v));
1200 : instrument_atomic_read_write(old, sizeof(*old));
1201 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1202 : }
1203 :
1204 : static __always_inline bool
1205 : atomic64_sub_and_test(s64 i, atomic64_t *v)
1206 : {
1207 : kcsan_mb();
1208 : instrument_atomic_read_write(v, sizeof(*v));
1209 : return arch_atomic64_sub_and_test(i, v);
1210 : }
1211 :
1212 : static __always_inline bool
1213 : atomic64_dec_and_test(atomic64_t *v)
1214 : {
1215 : kcsan_mb();
1216 : instrument_atomic_read_write(v, sizeof(*v));
1217 : return arch_atomic64_dec_and_test(v);
1218 : }
1219 :
1220 : static __always_inline bool
1221 : atomic64_inc_and_test(atomic64_t *v)
1222 : {
1223 : kcsan_mb();
1224 : instrument_atomic_read_write(v, sizeof(*v));
1225 : return arch_atomic64_inc_and_test(v);
1226 : }
1227 :
1228 : static __always_inline bool
1229 : atomic64_add_negative(s64 i, atomic64_t *v)
1230 : {
1231 : kcsan_mb();
1232 : instrument_atomic_read_write(v, sizeof(*v));
1233 : return arch_atomic64_add_negative(i, v);
1234 : }
1235 :
1236 : static __always_inline bool
1237 : atomic64_add_negative_acquire(s64 i, atomic64_t *v)
1238 : {
1239 : instrument_atomic_read_write(v, sizeof(*v));
1240 : return arch_atomic64_add_negative_acquire(i, v);
1241 : }
1242 :
1243 : static __always_inline bool
1244 : atomic64_add_negative_release(s64 i, atomic64_t *v)
1245 : {
1246 : kcsan_release();
1247 : instrument_atomic_read_write(v, sizeof(*v));
1248 : return arch_atomic64_add_negative_release(i, v);
1249 : }
1250 :
1251 : static __always_inline bool
1252 : atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
1253 : {
1254 : instrument_atomic_read_write(v, sizeof(*v));
1255 : return arch_atomic64_add_negative_relaxed(i, v);
1256 : }
1257 :
1258 : static __always_inline s64
1259 : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1260 : {
1261 : kcsan_mb();
1262 : instrument_atomic_read_write(v, sizeof(*v));
1263 : return arch_atomic64_fetch_add_unless(v, a, u);
1264 : }
1265 :
1266 : static __always_inline bool
1267 : atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1268 : {
1269 : kcsan_mb();
1270 : instrument_atomic_read_write(v, sizeof(*v));
1271 : return arch_atomic64_add_unless(v, a, u);
1272 : }
1273 :
1274 : static __always_inline bool
1275 : atomic64_inc_not_zero(atomic64_t *v)
1276 : {
1277 : kcsan_mb();
1278 : instrument_atomic_read_write(v, sizeof(*v));
1279 : return arch_atomic64_inc_not_zero(v);
1280 : }
1281 :
1282 : static __always_inline bool
1283 : atomic64_inc_unless_negative(atomic64_t *v)
1284 : {
1285 : kcsan_mb();
1286 : instrument_atomic_read_write(v, sizeof(*v));
1287 : return arch_atomic64_inc_unless_negative(v);
1288 : }
1289 :
1290 : static __always_inline bool
1291 : atomic64_dec_unless_positive(atomic64_t *v)
1292 : {
1293 : kcsan_mb();
1294 : instrument_atomic_read_write(v, sizeof(*v));
1295 : return arch_atomic64_dec_unless_positive(v);
1296 : }
1297 :
1298 : static __always_inline s64
1299 : atomic64_dec_if_positive(atomic64_t *v)
1300 : {
1301 : kcsan_mb();
1302 : instrument_atomic_read_write(v, sizeof(*v));
1303 : return arch_atomic64_dec_if_positive(v);
1304 : }
1305 :
1306 : static __always_inline long
1307 : atomic_long_read(const atomic_long_t *v)
1308 : {
1309 32814 : instrument_atomic_read(v, sizeof(*v));
1310 32814 : return arch_atomic_long_read(v);
1311 : }
1312 :
1313 : static __always_inline long
1314 : atomic_long_read_acquire(const atomic_long_t *v)
1315 : {
1316 : instrument_atomic_read(v, sizeof(*v));
1317 : return arch_atomic_long_read_acquire(v);
1318 : }
1319 :
1320 : static __always_inline void
1321 : atomic_long_set(atomic_long_t *v, long i)
1322 : {
1323 69828 : instrument_atomic_write(v, sizeof(*v));
1324 69828 : arch_atomic_long_set(v, i);
1325 : }
1326 :
1327 : static __always_inline void
1328 : atomic_long_set_release(atomic_long_t *v, long i)
1329 : {
1330 : kcsan_release();
1331 : instrument_atomic_write(v, sizeof(*v));
1332 : arch_atomic_long_set_release(v, i);
1333 : }
1334 :
1335 : static __always_inline void
1336 : atomic_long_add(long i, atomic_long_t *v)
1337 : {
1338 19159 : instrument_atomic_read_write(v, sizeof(*v));
1339 19159 : arch_atomic_long_add(i, v);
1340 : }
1341 :
1342 : static __always_inline long
1343 : atomic_long_add_return(long i, atomic_long_t *v)
1344 : {
1345 : kcsan_mb();
1346 641 : instrument_atomic_read_write(v, sizeof(*v));
1347 641 : return arch_atomic_long_add_return(i, v);
1348 : }
1349 :
1350 : static __always_inline long
1351 : atomic_long_add_return_acquire(long i, atomic_long_t *v)
1352 : {
1353 731 : instrument_atomic_read_write(v, sizeof(*v));
1354 731 : return arch_atomic_long_add_return_acquire(i, v);
1355 : }
1356 :
1357 : static __always_inline long
1358 : atomic_long_add_return_release(long i, atomic_long_t *v)
1359 : {
1360 : kcsan_release();
1361 731 : instrument_atomic_read_write(v, sizeof(*v));
1362 731 : return arch_atomic_long_add_return_release(i, v);
1363 : }
1364 :
1365 : static __always_inline long
1366 : atomic_long_add_return_relaxed(long i, atomic_long_t *v)
1367 : {
1368 : instrument_atomic_read_write(v, sizeof(*v));
1369 : return arch_atomic_long_add_return_relaxed(i, v);
1370 : }
1371 :
1372 : static __always_inline long
1373 : atomic_long_fetch_add(long i, atomic_long_t *v)
1374 : {
1375 : kcsan_mb();
1376 0 : instrument_atomic_read_write(v, sizeof(*v));
1377 0 : return arch_atomic_long_fetch_add(i, v);
1378 : }
1379 :
1380 : static __always_inline long
1381 : atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
1382 : {
1383 : instrument_atomic_read_write(v, sizeof(*v));
1384 : return arch_atomic_long_fetch_add_acquire(i, v);
1385 : }
1386 :
1387 : static __always_inline long
1388 : atomic_long_fetch_add_release(long i, atomic_long_t *v)
1389 : {
1390 : kcsan_release();
1391 27889 : instrument_atomic_read_write(v, sizeof(*v));
1392 27889 : return arch_atomic_long_fetch_add_release(i, v);
1393 : }
1394 :
1395 : static __always_inline long
1396 : atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
1397 : {
1398 : instrument_atomic_read_write(v, sizeof(*v));
1399 : return arch_atomic_long_fetch_add_relaxed(i, v);
1400 : }
1401 :
1402 : static __always_inline void
1403 : atomic_long_sub(long i, atomic_long_t *v)
1404 : {
1405 2016 : instrument_atomic_read_write(v, sizeof(*v));
1406 2016 : arch_atomic_long_sub(i, v);
1407 : }
1408 :
1409 : static __always_inline long
1410 : atomic_long_sub_return(long i, atomic_long_t *v)
1411 : {
1412 : kcsan_mb();
1413 367 : instrument_atomic_read_write(v, sizeof(*v));
1414 367 : return arch_atomic_long_sub_return(i, v);
1415 : }
1416 :
1417 : static __always_inline long
1418 : atomic_long_sub_return_acquire(long i, atomic_long_t *v)
1419 : {
1420 : instrument_atomic_read_write(v, sizeof(*v));
1421 : return arch_atomic_long_sub_return_acquire(i, v);
1422 : }
1423 :
1424 : static __always_inline long
1425 : atomic_long_sub_return_release(long i, atomic_long_t *v)
1426 : {
1427 : kcsan_release();
1428 : instrument_atomic_read_write(v, sizeof(*v));
1429 : return arch_atomic_long_sub_return_release(i, v);
1430 : }
1431 :
1432 : static __always_inline long
1433 : atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
1434 : {
1435 : instrument_atomic_read_write(v, sizeof(*v));
1436 : return arch_atomic_long_sub_return_relaxed(i, v);
1437 : }
1438 :
1439 : static __always_inline long
1440 : atomic_long_fetch_sub(long i, atomic_long_t *v)
1441 : {
1442 : kcsan_mb();
1443 : instrument_atomic_read_write(v, sizeof(*v));
1444 : return arch_atomic_long_fetch_sub(i, v);
1445 : }
1446 :
1447 : static __always_inline long
1448 : atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
1449 : {
1450 : instrument_atomic_read_write(v, sizeof(*v));
1451 : return arch_atomic_long_fetch_sub_acquire(i, v);
1452 : }
1453 :
1454 : static __always_inline long
1455 : atomic_long_fetch_sub_release(long i, atomic_long_t *v)
1456 : {
1457 : kcsan_release();
1458 : instrument_atomic_read_write(v, sizeof(*v));
1459 : return arch_atomic_long_fetch_sub_release(i, v);
1460 : }
1461 :
1462 : static __always_inline long
1463 : atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
1464 : {
1465 : instrument_atomic_read_write(v, sizeof(*v));
1466 : return arch_atomic_long_fetch_sub_relaxed(i, v);
1467 : }
1468 :
1469 : static __always_inline void
1470 : atomic_long_inc(atomic_long_t *v)
1471 : {
1472 2194 : instrument_atomic_read_write(v, sizeof(*v));
1473 2194 : arch_atomic_long_inc(v);
1474 : }
1475 :
1476 : static __always_inline long
1477 : atomic_long_inc_return(atomic_long_t *v)
1478 : {
1479 : kcsan_mb();
1480 0 : instrument_atomic_read_write(v, sizeof(*v));
1481 0 : return arch_atomic_long_inc_return(v);
1482 : }
1483 :
1484 : static __always_inline long
1485 : atomic_long_inc_return_acquire(atomic_long_t *v)
1486 : {
1487 : instrument_atomic_read_write(v, sizeof(*v));
1488 : return arch_atomic_long_inc_return_acquire(v);
1489 : }
1490 :
1491 : static __always_inline long
1492 : atomic_long_inc_return_release(atomic_long_t *v)
1493 : {
1494 : kcsan_release();
1495 : instrument_atomic_read_write(v, sizeof(*v));
1496 : return arch_atomic_long_inc_return_release(v);
1497 : }
1498 :
1499 : static __always_inline long
1500 : atomic_long_inc_return_relaxed(atomic_long_t *v)
1501 : {
1502 10 : instrument_atomic_read_write(v, sizeof(*v));
1503 10 : return arch_atomic_long_inc_return_relaxed(v);
1504 : }
1505 :
1506 : static __always_inline long
1507 : atomic_long_fetch_inc(atomic_long_t *v)
1508 : {
1509 : kcsan_mb();
1510 : instrument_atomic_read_write(v, sizeof(*v));
1511 : return arch_atomic_long_fetch_inc(v);
1512 : }
1513 :
1514 : static __always_inline long
1515 : atomic_long_fetch_inc_acquire(atomic_long_t *v)
1516 : {
1517 : instrument_atomic_read_write(v, sizeof(*v));
1518 : return arch_atomic_long_fetch_inc_acquire(v);
1519 : }
1520 :
1521 : static __always_inline long
1522 : atomic_long_fetch_inc_release(atomic_long_t *v)
1523 : {
1524 : kcsan_release();
1525 : instrument_atomic_read_write(v, sizeof(*v));
1526 : return arch_atomic_long_fetch_inc_release(v);
1527 : }
1528 :
1529 : static __always_inline long
1530 : atomic_long_fetch_inc_relaxed(atomic_long_t *v)
1531 : {
1532 : instrument_atomic_read_write(v, sizeof(*v));
1533 : return arch_atomic_long_fetch_inc_relaxed(v);
1534 : }
1535 :
1536 : static __always_inline void
1537 : atomic_long_dec(atomic_long_t *v)
1538 : {
1539 1753 : instrument_atomic_read_write(v, sizeof(*v));
1540 1753 : arch_atomic_long_dec(v);
1541 : }
1542 :
1543 : static __always_inline long
1544 : atomic_long_dec_return(atomic_long_t *v)
1545 : {
1546 : kcsan_mb();
1547 : instrument_atomic_read_write(v, sizeof(*v));
1548 : return arch_atomic_long_dec_return(v);
1549 : }
1550 :
1551 : static __always_inline long
1552 : atomic_long_dec_return_acquire(atomic_long_t *v)
1553 : {
1554 : instrument_atomic_read_write(v, sizeof(*v));
1555 : return arch_atomic_long_dec_return_acquire(v);
1556 : }
1557 :
1558 : static __always_inline long
1559 : atomic_long_dec_return_release(atomic_long_t *v)
1560 : {
1561 : kcsan_release();
1562 : instrument_atomic_read_write(v, sizeof(*v));
1563 : return arch_atomic_long_dec_return_release(v);
1564 : }
1565 :
1566 : static __always_inline long
1567 : atomic_long_dec_return_relaxed(atomic_long_t *v)
1568 : {
1569 : instrument_atomic_read_write(v, sizeof(*v));
1570 : return arch_atomic_long_dec_return_relaxed(v);
1571 : }
1572 :
1573 : static __always_inline long
1574 : atomic_long_fetch_dec(atomic_long_t *v)
1575 : {
1576 : kcsan_mb();
1577 : instrument_atomic_read_write(v, sizeof(*v));
1578 : return arch_atomic_long_fetch_dec(v);
1579 : }
1580 :
1581 : static __always_inline long
1582 : atomic_long_fetch_dec_acquire(atomic_long_t *v)
1583 : {
1584 : instrument_atomic_read_write(v, sizeof(*v));
1585 : return arch_atomic_long_fetch_dec_acquire(v);
1586 : }
1587 :
1588 : static __always_inline long
1589 : atomic_long_fetch_dec_release(atomic_long_t *v)
1590 : {
1591 : kcsan_release();
1592 : instrument_atomic_read_write(v, sizeof(*v));
1593 : return arch_atomic_long_fetch_dec_release(v);
1594 : }
1595 :
1596 : static __always_inline long
1597 : atomic_long_fetch_dec_relaxed(atomic_long_t *v)
1598 : {
1599 : instrument_atomic_read_write(v, sizeof(*v));
1600 : return arch_atomic_long_fetch_dec_relaxed(v);
1601 : }
1602 :
1603 : static __always_inline void
1604 : atomic_long_and(long i, atomic_long_t *v)
1605 : {
1606 : instrument_atomic_read_write(v, sizeof(*v));
1607 : arch_atomic_long_and(i, v);
1608 : }
1609 :
1610 : static __always_inline long
1611 : atomic_long_fetch_and(long i, atomic_long_t *v)
1612 : {
1613 : kcsan_mb();
1614 : instrument_atomic_read_write(v, sizeof(*v));
1615 : return arch_atomic_long_fetch_and(i, v);
1616 : }
1617 :
1618 : static __always_inline long
1619 : atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
1620 : {
1621 : instrument_atomic_read_write(v, sizeof(*v));
1622 : return arch_atomic_long_fetch_and_acquire(i, v);
1623 : }
1624 :
1625 : static __always_inline long
1626 : atomic_long_fetch_and_release(long i, atomic_long_t *v)
1627 : {
1628 : kcsan_release();
1629 : instrument_atomic_read_write(v, sizeof(*v));
1630 : return arch_atomic_long_fetch_and_release(i, v);
1631 : }
1632 :
1633 : static __always_inline long
1634 : atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
1635 : {
1636 : instrument_atomic_read_write(v, sizeof(*v));
1637 : return arch_atomic_long_fetch_and_relaxed(i, v);
1638 : }
1639 :
1640 : static __always_inline void
1641 : atomic_long_andnot(long i, atomic_long_t *v)
1642 : {
1643 0 : instrument_atomic_read_write(v, sizeof(*v));
1644 0 : arch_atomic_long_andnot(i, v);
1645 : }
1646 :
1647 : static __always_inline long
1648 : atomic_long_fetch_andnot(long i, atomic_long_t *v)
1649 : {
1650 : kcsan_mb();
1651 : instrument_atomic_read_write(v, sizeof(*v));
1652 : return arch_atomic_long_fetch_andnot(i, v);
1653 : }
1654 :
1655 : static __always_inline long
1656 : atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
1657 : {
1658 : instrument_atomic_read_write(v, sizeof(*v));
1659 : return arch_atomic_long_fetch_andnot_acquire(i, v);
1660 : }
1661 :
1662 : static __always_inline long
1663 : atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
1664 : {
1665 : kcsan_release();
1666 : instrument_atomic_read_write(v, sizeof(*v));
1667 : return arch_atomic_long_fetch_andnot_release(i, v);
1668 : }
1669 :
1670 : static __always_inline long
1671 : atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
1672 : {
1673 : instrument_atomic_read_write(v, sizeof(*v));
1674 : return arch_atomic_long_fetch_andnot_relaxed(i, v);
1675 : }
1676 :
1677 : static __always_inline void
1678 : atomic_long_or(long i, atomic_long_t *v)
1679 : {
1680 0 : instrument_atomic_read_write(v, sizeof(*v));
1681 0 : arch_atomic_long_or(i, v);
1682 : }
1683 :
1684 : static __always_inline long
1685 : atomic_long_fetch_or(long i, atomic_long_t *v)
1686 : {
1687 : kcsan_mb();
1688 : instrument_atomic_read_write(v, sizeof(*v));
1689 : return arch_atomic_long_fetch_or(i, v);
1690 : }
1691 :
1692 : static __always_inline long
1693 : atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
1694 : {
1695 : instrument_atomic_read_write(v, sizeof(*v));
1696 : return arch_atomic_long_fetch_or_acquire(i, v);
1697 : }
1698 :
1699 : static __always_inline long
1700 : atomic_long_fetch_or_release(long i, atomic_long_t *v)
1701 : {
1702 : kcsan_release();
1703 : instrument_atomic_read_write(v, sizeof(*v));
1704 : return arch_atomic_long_fetch_or_release(i, v);
1705 : }
1706 :
1707 : static __always_inline long
1708 : atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
1709 : {
1710 : instrument_atomic_read_write(v, sizeof(*v));
1711 : return arch_atomic_long_fetch_or_relaxed(i, v);
1712 : }
1713 :
1714 : static __always_inline void
1715 : atomic_long_xor(long i, atomic_long_t *v)
1716 : {
1717 : instrument_atomic_read_write(v, sizeof(*v));
1718 : arch_atomic_long_xor(i, v);
1719 : }
1720 :
1721 : static __always_inline long
1722 : atomic_long_fetch_xor(long i, atomic_long_t *v)
1723 : {
1724 : kcsan_mb();
1725 : instrument_atomic_read_write(v, sizeof(*v));
1726 : return arch_atomic_long_fetch_xor(i, v);
1727 : }
1728 :
1729 : static __always_inline long
1730 : atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
1731 : {
1732 : instrument_atomic_read_write(v, sizeof(*v));
1733 : return arch_atomic_long_fetch_xor_acquire(i, v);
1734 : }
1735 :
1736 : static __always_inline long
1737 : atomic_long_fetch_xor_release(long i, atomic_long_t *v)
1738 : {
1739 : kcsan_release();
1740 : instrument_atomic_read_write(v, sizeof(*v));
1741 : return arch_atomic_long_fetch_xor_release(i, v);
1742 : }
1743 :
1744 : static __always_inline long
1745 : atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
1746 : {
1747 : instrument_atomic_read_write(v, sizeof(*v));
1748 : return arch_atomic_long_fetch_xor_relaxed(i, v);
1749 : }
1750 :
1751 : static __always_inline long
1752 : atomic_long_xchg(atomic_long_t *v, long i)
1753 : {
1754 : kcsan_mb();
1755 0 : instrument_atomic_read_write(v, sizeof(*v));
1756 0 : return arch_atomic_long_xchg(v, i);
1757 : }
1758 :
1759 : static __always_inline long
1760 : atomic_long_xchg_acquire(atomic_long_t *v, long i)
1761 : {
1762 : instrument_atomic_read_write(v, sizeof(*v));
1763 : return arch_atomic_long_xchg_acquire(v, i);
1764 : }
1765 :
1766 : static __always_inline long
1767 : atomic_long_xchg_release(atomic_long_t *v, long i)
1768 : {
1769 : kcsan_release();
1770 : instrument_atomic_read_write(v, sizeof(*v));
1771 : return arch_atomic_long_xchg_release(v, i);
1772 : }
1773 :
1774 : static __always_inline long
1775 : atomic_long_xchg_relaxed(atomic_long_t *v, long i)
1776 : {
1777 : instrument_atomic_read_write(v, sizeof(*v));
1778 : return arch_atomic_long_xchg_relaxed(v, i);
1779 : }
1780 :
1781 : static __always_inline long
1782 : atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
1783 : {
1784 : kcsan_mb();
1785 1 : instrument_atomic_read_write(v, sizeof(*v));
1786 1 : return arch_atomic_long_cmpxchg(v, old, new);
1787 : }
1788 :
1789 : static __always_inline long
1790 : atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
1791 : {
1792 : instrument_atomic_read_write(v, sizeof(*v));
1793 : return arch_atomic_long_cmpxchg_acquire(v, old, new);
1794 : }
1795 :
1796 : static __always_inline long
1797 : atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
1798 : {
1799 : kcsan_release();
1800 : instrument_atomic_read_write(v, sizeof(*v));
1801 : return arch_atomic_long_cmpxchg_release(v, old, new);
1802 : }
1803 :
1804 : static __always_inline long
1805 : atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
1806 : {
1807 900 : instrument_atomic_read_write(v, sizeof(*v));
1808 900 : return arch_atomic_long_cmpxchg_relaxed(v, old, new);
1809 : }
1810 :
1811 : static __always_inline bool
1812 : atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
1813 : {
1814 : kcsan_mb();
1815 3780 : instrument_atomic_read_write(v, sizeof(*v));
1816 3780 : instrument_atomic_read_write(old, sizeof(*old));
1817 3780 : return arch_atomic_long_try_cmpxchg(v, old, new);
1818 : }
1819 :
1820 : static __always_inline bool
1821 : atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
1822 : {
1823 33818 : instrument_atomic_read_write(v, sizeof(*v));
1824 33818 : instrument_atomic_read_write(old, sizeof(*old));
1825 33818 : return arch_atomic_long_try_cmpxchg_acquire(v, old, new);
1826 : }
1827 :
1828 : static __always_inline bool
1829 : atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
1830 : {
1831 : kcsan_release();
1832 5923 : instrument_atomic_read_write(v, sizeof(*v));
1833 5923 : instrument_atomic_read_write(old, sizeof(*old));
1834 5923 : return arch_atomic_long_try_cmpxchg_release(v, old, new);
1835 : }
1836 :
1837 : static __always_inline bool
1838 : atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
1839 : {
1840 : instrument_atomic_read_write(v, sizeof(*v));
1841 : instrument_atomic_read_write(old, sizeof(*old));
1842 : return arch_atomic_long_try_cmpxchg_relaxed(v, old, new);
1843 : }
1844 :
1845 : static __always_inline bool
1846 : atomic_long_sub_and_test(long i, atomic_long_t *v)
1847 : {
1848 : kcsan_mb();
1849 0 : instrument_atomic_read_write(v, sizeof(*v));
1850 0 : return arch_atomic_long_sub_and_test(i, v);
1851 : }
1852 :
1853 : static __always_inline bool
1854 : atomic_long_dec_and_test(atomic_long_t *v)
1855 : {
1856 : kcsan_mb();
1857 0 : instrument_atomic_read_write(v, sizeof(*v));
1858 0 : return arch_atomic_long_dec_and_test(v);
1859 : }
1860 :
1861 : static __always_inline bool
1862 : atomic_long_inc_and_test(atomic_long_t *v)
1863 : {
1864 : kcsan_mb();
1865 : instrument_atomic_read_write(v, sizeof(*v));
1866 : return arch_atomic_long_inc_and_test(v);
1867 : }
1868 :
1869 : static __always_inline bool
1870 : atomic_long_add_negative(long i, atomic_long_t *v)
1871 : {
1872 : kcsan_mb();
1873 : instrument_atomic_read_write(v, sizeof(*v));
1874 : return arch_atomic_long_add_negative(i, v);
1875 : }
1876 :
1877 : static __always_inline bool
1878 : atomic_long_add_negative_acquire(long i, atomic_long_t *v)
1879 : {
1880 : instrument_atomic_read_write(v, sizeof(*v));
1881 : return arch_atomic_long_add_negative_acquire(i, v);
1882 : }
1883 :
1884 : static __always_inline bool
1885 : atomic_long_add_negative_release(long i, atomic_long_t *v)
1886 : {
1887 : kcsan_release();
1888 : instrument_atomic_read_write(v, sizeof(*v));
1889 : return arch_atomic_long_add_negative_release(i, v);
1890 : }
1891 :
1892 : static __always_inline bool
1893 : atomic_long_add_negative_relaxed(long i, atomic_long_t *v)
1894 : {
1895 : instrument_atomic_read_write(v, sizeof(*v));
1896 : return arch_atomic_long_add_negative_relaxed(i, v);
1897 : }
1898 :
1899 : static __always_inline long
1900 : atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
1901 : {
1902 : kcsan_mb();
1903 : instrument_atomic_read_write(v, sizeof(*v));
1904 : return arch_atomic_long_fetch_add_unless(v, a, u);
1905 : }
1906 :
1907 : static __always_inline bool
1908 : atomic_long_add_unless(atomic_long_t *v, long a, long u)
1909 : {
1910 : kcsan_mb();
1911 0 : instrument_atomic_read_write(v, sizeof(*v));
1912 0 : return arch_atomic_long_add_unless(v, a, u);
1913 : }
1914 :
1915 : static __always_inline bool
1916 : atomic_long_inc_not_zero(atomic_long_t *v)
1917 : {
1918 : kcsan_mb();
1919 0 : instrument_atomic_read_write(v, sizeof(*v));
1920 0 : return arch_atomic_long_inc_not_zero(v);
1921 : }
1922 :
1923 : static __always_inline bool
1924 : atomic_long_inc_unless_negative(atomic_long_t *v)
1925 : {
1926 : kcsan_mb();
1927 : instrument_atomic_read_write(v, sizeof(*v));
1928 : return arch_atomic_long_inc_unless_negative(v);
1929 : }
1930 :
1931 : static __always_inline bool
1932 : atomic_long_dec_unless_positive(atomic_long_t *v)
1933 : {
1934 : kcsan_mb();
1935 : instrument_atomic_read_write(v, sizeof(*v));
1936 : return arch_atomic_long_dec_unless_positive(v);
1937 : }
1938 :
1939 : static __always_inline long
1940 : atomic_long_dec_if_positive(atomic_long_t *v)
1941 : {
1942 : kcsan_mb();
1943 0 : instrument_atomic_read_write(v, sizeof(*v));
1944 0 : return arch_atomic_long_dec_if_positive(v);
1945 : }
1946 :
1947 : #define xchg(ptr, ...) \
1948 : ({ \
1949 : typeof(ptr) __ai_ptr = (ptr); \
1950 : kcsan_mb(); \
1951 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1952 : arch_xchg(__ai_ptr, __VA_ARGS__); \
1953 : })
1954 :
1955 : #define xchg_acquire(ptr, ...) \
1956 : ({ \
1957 : typeof(ptr) __ai_ptr = (ptr); \
1958 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1959 : arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1960 : })
1961 :
1962 : #define xchg_release(ptr, ...) \
1963 : ({ \
1964 : typeof(ptr) __ai_ptr = (ptr); \
1965 : kcsan_release(); \
1966 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1967 : arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1968 : })
1969 :
1970 : #define xchg_relaxed(ptr, ...) \
1971 : ({ \
1972 : typeof(ptr) __ai_ptr = (ptr); \
1973 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1974 : arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1975 : })
1976 :
1977 : #define cmpxchg(ptr, ...) \
1978 : ({ \
1979 : typeof(ptr) __ai_ptr = (ptr); \
1980 : kcsan_mb(); \
1981 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1982 : arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1983 : })
1984 :
1985 : #define cmpxchg_acquire(ptr, ...) \
1986 : ({ \
1987 : typeof(ptr) __ai_ptr = (ptr); \
1988 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1989 : arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1990 : })
1991 :
1992 : #define cmpxchg_release(ptr, ...) \
1993 : ({ \
1994 : typeof(ptr) __ai_ptr = (ptr); \
1995 : kcsan_release(); \
1996 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
1997 : arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1998 : })
1999 :
2000 : #define cmpxchg_relaxed(ptr, ...) \
2001 : ({ \
2002 : typeof(ptr) __ai_ptr = (ptr); \
2003 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2004 : arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
2005 : })
2006 :
2007 : #define cmpxchg64(ptr, ...) \
2008 : ({ \
2009 : typeof(ptr) __ai_ptr = (ptr); \
2010 : kcsan_mb(); \
2011 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2012 : arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
2013 : })
2014 :
2015 : #define cmpxchg64_acquire(ptr, ...) \
2016 : ({ \
2017 : typeof(ptr) __ai_ptr = (ptr); \
2018 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2019 : arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
2020 : })
2021 :
2022 : #define cmpxchg64_release(ptr, ...) \
2023 : ({ \
2024 : typeof(ptr) __ai_ptr = (ptr); \
2025 : kcsan_release(); \
2026 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2027 : arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
2028 : })
2029 :
2030 : #define cmpxchg64_relaxed(ptr, ...) \
2031 : ({ \
2032 : typeof(ptr) __ai_ptr = (ptr); \
2033 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2034 : arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
2035 : })
2036 :
2037 : #define try_cmpxchg(ptr, oldp, ...) \
2038 : ({ \
2039 : typeof(ptr) __ai_ptr = (ptr); \
2040 : typeof(oldp) __ai_oldp = (oldp); \
2041 : kcsan_mb(); \
2042 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2043 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2044 : arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2045 : })
2046 :
2047 : #define try_cmpxchg_acquire(ptr, oldp, ...) \
2048 : ({ \
2049 : typeof(ptr) __ai_ptr = (ptr); \
2050 : typeof(oldp) __ai_oldp = (oldp); \
2051 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2052 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2053 : arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2054 : })
2055 :
2056 : #define try_cmpxchg_release(ptr, oldp, ...) \
2057 : ({ \
2058 : typeof(ptr) __ai_ptr = (ptr); \
2059 : typeof(oldp) __ai_oldp = (oldp); \
2060 : kcsan_release(); \
2061 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2062 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2063 : arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2064 : })
2065 :
2066 : #define try_cmpxchg_relaxed(ptr, oldp, ...) \
2067 : ({ \
2068 : typeof(ptr) __ai_ptr = (ptr); \
2069 : typeof(oldp) __ai_oldp = (oldp); \
2070 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2071 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2072 : arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2073 : })
2074 :
2075 : #define try_cmpxchg64(ptr, oldp, ...) \
2076 : ({ \
2077 : typeof(ptr) __ai_ptr = (ptr); \
2078 : typeof(oldp) __ai_oldp = (oldp); \
2079 : kcsan_mb(); \
2080 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2081 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2082 : arch_try_cmpxchg64(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2083 : })
2084 :
2085 : #define try_cmpxchg64_acquire(ptr, oldp, ...) \
2086 : ({ \
2087 : typeof(ptr) __ai_ptr = (ptr); \
2088 : typeof(oldp) __ai_oldp = (oldp); \
2089 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2090 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2091 : arch_try_cmpxchg64_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2092 : })
2093 :
2094 : #define try_cmpxchg64_release(ptr, oldp, ...) \
2095 : ({ \
2096 : typeof(ptr) __ai_ptr = (ptr); \
2097 : typeof(oldp) __ai_oldp = (oldp); \
2098 : kcsan_release(); \
2099 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2100 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2101 : arch_try_cmpxchg64_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2102 : })
2103 :
2104 : #define try_cmpxchg64_relaxed(ptr, oldp, ...) \
2105 : ({ \
2106 : typeof(ptr) __ai_ptr = (ptr); \
2107 : typeof(oldp) __ai_oldp = (oldp); \
2108 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2109 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2110 : arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2111 : })
2112 :
2113 : #define cmpxchg_local(ptr, ...) \
2114 : ({ \
2115 : typeof(ptr) __ai_ptr = (ptr); \
2116 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2117 : arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
2118 : })
2119 :
2120 : #define cmpxchg64_local(ptr, ...) \
2121 : ({ \
2122 : typeof(ptr) __ai_ptr = (ptr); \
2123 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2124 : arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
2125 : })
2126 :
2127 : #define sync_cmpxchg(ptr, ...) \
2128 : ({ \
2129 : typeof(ptr) __ai_ptr = (ptr); \
2130 : kcsan_mb(); \
2131 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2132 : arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
2133 : })
2134 :
2135 : #define try_cmpxchg_local(ptr, oldp, ...) \
2136 : ({ \
2137 : typeof(ptr) __ai_ptr = (ptr); \
2138 : typeof(oldp) __ai_oldp = (oldp); \
2139 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2140 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2141 : arch_try_cmpxchg_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2142 : })
2143 :
2144 : #define try_cmpxchg64_local(ptr, oldp, ...) \
2145 : ({ \
2146 : typeof(ptr) __ai_ptr = (ptr); \
2147 : typeof(oldp) __ai_oldp = (oldp); \
2148 : instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
2149 : instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
2150 : arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
2151 : })
2152 :
2153 : #define cmpxchg_double(ptr, ...) \
2154 : ({ \
2155 : typeof(ptr) __ai_ptr = (ptr); \
2156 : kcsan_mb(); \
2157 : instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2158 : arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
2159 : })
2160 :
2161 :
2162 : #define cmpxchg_double_local(ptr, ...) \
2163 : ({ \
2164 : typeof(ptr) __ai_ptr = (ptr); \
2165 : instrument_atomic_read_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
2166 : arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
2167 : })
2168 :
2169 : #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
2170 : // 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
|
