Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-fallback.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : #ifndef _LINUX_ATOMIC_FALLBACK_H
7 : #define _LINUX_ATOMIC_FALLBACK_H
8 :
9 : #include <linux/compiler.h>
10 :
11 : #ifndef arch_xchg_relaxed
12 : #define arch_xchg_acquire arch_xchg
13 : #define arch_xchg_release arch_xchg
14 : #define arch_xchg_relaxed arch_xchg
15 : #else /* arch_xchg_relaxed */
16 :
17 : #ifndef arch_xchg_acquire
18 : #define arch_xchg_acquire(...) \
19 : __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20 : #endif
21 :
22 : #ifndef arch_xchg_release
23 : #define arch_xchg_release(...) \
24 : __atomic_op_release(arch_xchg, __VA_ARGS__)
25 : #endif
26 :
27 : #ifndef arch_xchg
28 : #define arch_xchg(...) \
29 : __atomic_op_fence(arch_xchg, __VA_ARGS__)
30 : #endif
31 :
32 : #endif /* arch_xchg_relaxed */
33 :
34 : #ifndef arch_cmpxchg_relaxed
35 : #define arch_cmpxchg_acquire arch_cmpxchg
36 : #define arch_cmpxchg_release arch_cmpxchg
37 : #define arch_cmpxchg_relaxed arch_cmpxchg
38 : #else /* arch_cmpxchg_relaxed */
39 :
40 : #ifndef arch_cmpxchg_acquire
41 : #define arch_cmpxchg_acquire(...) \
42 : __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43 : #endif
44 :
45 : #ifndef arch_cmpxchg_release
46 : #define arch_cmpxchg_release(...) \
47 : __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48 : #endif
49 :
50 : #ifndef arch_cmpxchg
51 : #define arch_cmpxchg(...) \
52 : __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53 : #endif
54 :
55 : #endif /* arch_cmpxchg_relaxed */
56 :
57 : #ifndef arch_cmpxchg64_relaxed
58 : #define arch_cmpxchg64_acquire arch_cmpxchg64
59 : #define arch_cmpxchg64_release arch_cmpxchg64
60 : #define arch_cmpxchg64_relaxed arch_cmpxchg64
61 : #else /* arch_cmpxchg64_relaxed */
62 :
63 : #ifndef arch_cmpxchg64_acquire
64 : #define arch_cmpxchg64_acquire(...) \
65 : __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66 : #endif
67 :
68 : #ifndef arch_cmpxchg64_release
69 : #define arch_cmpxchg64_release(...) \
70 : __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71 : #endif
72 :
73 : #ifndef arch_cmpxchg64
74 : #define arch_cmpxchg64(...) \
75 : __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76 : #endif
77 :
78 : #endif /* arch_cmpxchg64_relaxed */
79 :
80 : #ifndef arch_try_cmpxchg_relaxed
81 : #ifdef arch_try_cmpxchg
82 : #define arch_try_cmpxchg_acquire arch_try_cmpxchg
83 : #define arch_try_cmpxchg_release arch_try_cmpxchg
84 : #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85 : #endif /* arch_try_cmpxchg */
86 :
87 : #ifndef arch_try_cmpxchg
88 : #define arch_try_cmpxchg(_ptr, _oldp, _new) \
89 : ({ \
90 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 : ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
92 : if (unlikely(___r != ___o)) \
93 : *___op = ___r; \
94 : likely(___r == ___o); \
95 : })
96 : #endif /* arch_try_cmpxchg */
97 :
98 : #ifndef arch_try_cmpxchg_acquire
99 : #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
100 : ({ \
101 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 : ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
103 : if (unlikely(___r != ___o)) \
104 : *___op = ___r; \
105 : likely(___r == ___o); \
106 : })
107 : #endif /* arch_try_cmpxchg_acquire */
108 :
109 : #ifndef arch_try_cmpxchg_release
110 : #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
111 : ({ \
112 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 : ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
114 : if (unlikely(___r != ___o)) \
115 : *___op = ___r; \
116 : likely(___r == ___o); \
117 : })
118 : #endif /* arch_try_cmpxchg_release */
119 :
120 : #ifndef arch_try_cmpxchg_relaxed
121 : #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
122 : ({ \
123 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 : ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
125 : if (unlikely(___r != ___o)) \
126 : *___op = ___r; \
127 : likely(___r == ___o); \
128 : })
129 : #endif /* arch_try_cmpxchg_relaxed */
130 :
131 : #else /* arch_try_cmpxchg_relaxed */
132 :
133 : #ifndef arch_try_cmpxchg_acquire
134 : #define arch_try_cmpxchg_acquire(...) \
135 : __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136 : #endif
137 :
138 : #ifndef arch_try_cmpxchg_release
139 : #define arch_try_cmpxchg_release(...) \
140 : __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
141 : #endif
142 :
143 : #ifndef arch_try_cmpxchg
144 : #define arch_try_cmpxchg(...) \
145 : __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
146 : #endif
147 :
148 : #endif /* arch_try_cmpxchg_relaxed */
149 :
150 : #ifndef arch_try_cmpxchg64_relaxed
151 : #ifdef arch_try_cmpxchg64
152 : #define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
153 : #define arch_try_cmpxchg64_release arch_try_cmpxchg64
154 : #define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
155 : #endif /* arch_try_cmpxchg64 */
156 :
157 : #ifndef arch_try_cmpxchg64
158 : #define arch_try_cmpxchg64(_ptr, _oldp, _new) \
159 : ({ \
160 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
161 : ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
162 : if (unlikely(___r != ___o)) \
163 : *___op = ___r; \
164 : likely(___r == ___o); \
165 : })
166 : #endif /* arch_try_cmpxchg64 */
167 :
168 : #ifndef arch_try_cmpxchg64_acquire
169 : #define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
170 : ({ \
171 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
172 : ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
173 : if (unlikely(___r != ___o)) \
174 : *___op = ___r; \
175 : likely(___r == ___o); \
176 : })
177 : #endif /* arch_try_cmpxchg64_acquire */
178 :
179 : #ifndef arch_try_cmpxchg64_release
180 : #define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
181 : ({ \
182 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
183 : ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
184 : if (unlikely(___r != ___o)) \
185 : *___op = ___r; \
186 : likely(___r == ___o); \
187 : })
188 : #endif /* arch_try_cmpxchg64_release */
189 :
190 : #ifndef arch_try_cmpxchg64_relaxed
191 : #define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
192 : ({ \
193 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
194 : ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
195 : if (unlikely(___r != ___o)) \
196 : *___op = ___r; \
197 : likely(___r == ___o); \
198 : })
199 : #endif /* arch_try_cmpxchg64_relaxed */
200 :
201 : #else /* arch_try_cmpxchg64_relaxed */
202 :
203 : #ifndef arch_try_cmpxchg64_acquire
204 : #define arch_try_cmpxchg64_acquire(...) \
205 : __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
206 : #endif
207 :
208 : #ifndef arch_try_cmpxchg64_release
209 : #define arch_try_cmpxchg64_release(...) \
210 : __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
211 : #endif
212 :
213 : #ifndef arch_try_cmpxchg64
214 : #define arch_try_cmpxchg64(...) \
215 : __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
216 : #endif
217 :
218 : #endif /* arch_try_cmpxchg64_relaxed */
219 :
220 : #ifndef arch_try_cmpxchg_local
221 : #define arch_try_cmpxchg_local(_ptr, _oldp, _new) \
222 : ({ \
223 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
224 : ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \
225 : if (unlikely(___r != ___o)) \
226 : *___op = ___r; \
227 : likely(___r == ___o); \
228 : })
229 : #endif /* arch_try_cmpxchg_local */
230 :
231 : #ifndef arch_try_cmpxchg64_local
232 : #define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \
233 : ({ \
234 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
235 : ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \
236 : if (unlikely(___r != ___o)) \
237 : *___op = ___r; \
238 : likely(___r == ___o); \
239 : })
240 : #endif /* arch_try_cmpxchg64_local */
241 :
242 : #ifndef arch_atomic_read_acquire
243 : static __always_inline int
244 : arch_atomic_read_acquire(const atomic_t *v)
245 : {
246 : int ret;
247 :
248 : if (__native_word(atomic_t)) {
249 0 : ret = smp_load_acquire(&(v)->counter);
250 : } else {
251 : ret = arch_atomic_read(v);
252 : __atomic_acquire_fence();
253 : }
254 :
255 : return ret;
256 : }
257 : #define arch_atomic_read_acquire arch_atomic_read_acquire
258 : #endif
259 :
260 : #ifndef arch_atomic_set_release
261 : static __always_inline void
262 : arch_atomic_set_release(atomic_t *v, int i)
263 : {
264 : if (__native_word(atomic_t)) {
265 0 : smp_store_release(&(v)->counter, i);
266 : } else {
267 : __atomic_release_fence();
268 : arch_atomic_set(v, i);
269 : }
270 : }
271 : #define arch_atomic_set_release arch_atomic_set_release
272 : #endif
273 :
274 : #ifndef arch_atomic_add_return_relaxed
275 : #define arch_atomic_add_return_acquire arch_atomic_add_return
276 : #define arch_atomic_add_return_release arch_atomic_add_return
277 : #define arch_atomic_add_return_relaxed arch_atomic_add_return
278 : #else /* arch_atomic_add_return_relaxed */
279 :
280 : #ifndef arch_atomic_add_return_acquire
281 : static __always_inline int
282 : arch_atomic_add_return_acquire(int i, atomic_t *v)
283 : {
284 : int ret = arch_atomic_add_return_relaxed(i, v);
285 : __atomic_acquire_fence();
286 : return ret;
287 : }
288 : #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
289 : #endif
290 :
291 : #ifndef arch_atomic_add_return_release
292 : static __always_inline int
293 : arch_atomic_add_return_release(int i, atomic_t *v)
294 : {
295 : __atomic_release_fence();
296 : return arch_atomic_add_return_relaxed(i, v);
297 : }
298 : #define arch_atomic_add_return_release arch_atomic_add_return_release
299 : #endif
300 :
301 : #ifndef arch_atomic_add_return
302 : static __always_inline int
303 : arch_atomic_add_return(int i, atomic_t *v)
304 : {
305 : int ret;
306 : __atomic_pre_full_fence();
307 : ret = arch_atomic_add_return_relaxed(i, v);
308 : __atomic_post_full_fence();
309 : return ret;
310 : }
311 : #define arch_atomic_add_return arch_atomic_add_return
312 : #endif
313 :
314 : #endif /* arch_atomic_add_return_relaxed */
315 :
316 : #ifndef arch_atomic_fetch_add_relaxed
317 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
318 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add
319 : #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
320 : #else /* arch_atomic_fetch_add_relaxed */
321 :
322 : #ifndef arch_atomic_fetch_add_acquire
323 : static __always_inline int
324 : arch_atomic_fetch_add_acquire(int i, atomic_t *v)
325 : {
326 : int ret = arch_atomic_fetch_add_relaxed(i, v);
327 : __atomic_acquire_fence();
328 : return ret;
329 : }
330 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
331 : #endif
332 :
333 : #ifndef arch_atomic_fetch_add_release
334 : static __always_inline int
335 : arch_atomic_fetch_add_release(int i, atomic_t *v)
336 : {
337 : __atomic_release_fence();
338 : return arch_atomic_fetch_add_relaxed(i, v);
339 : }
340 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
341 : #endif
342 :
343 : #ifndef arch_atomic_fetch_add
344 : static __always_inline int
345 : arch_atomic_fetch_add(int i, atomic_t *v)
346 : {
347 : int ret;
348 : __atomic_pre_full_fence();
349 : ret = arch_atomic_fetch_add_relaxed(i, v);
350 : __atomic_post_full_fence();
351 : return ret;
352 : }
353 : #define arch_atomic_fetch_add arch_atomic_fetch_add
354 : #endif
355 :
356 : #endif /* arch_atomic_fetch_add_relaxed */
357 :
358 : #ifndef arch_atomic_sub_return_relaxed
359 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return
360 : #define arch_atomic_sub_return_release arch_atomic_sub_return
361 : #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
362 : #else /* arch_atomic_sub_return_relaxed */
363 :
364 : #ifndef arch_atomic_sub_return_acquire
365 : static __always_inline int
366 : arch_atomic_sub_return_acquire(int i, atomic_t *v)
367 : {
368 : int ret = arch_atomic_sub_return_relaxed(i, v);
369 : __atomic_acquire_fence();
370 : return ret;
371 : }
372 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
373 : #endif
374 :
375 : #ifndef arch_atomic_sub_return_release
376 : static __always_inline int
377 : arch_atomic_sub_return_release(int i, atomic_t *v)
378 : {
379 : __atomic_release_fence();
380 : return arch_atomic_sub_return_relaxed(i, v);
381 : }
382 : #define arch_atomic_sub_return_release arch_atomic_sub_return_release
383 : #endif
384 :
385 : #ifndef arch_atomic_sub_return
386 : static __always_inline int
387 : arch_atomic_sub_return(int i, atomic_t *v)
388 : {
389 : int ret;
390 : __atomic_pre_full_fence();
391 : ret = arch_atomic_sub_return_relaxed(i, v);
392 : __atomic_post_full_fence();
393 : return ret;
394 : }
395 : #define arch_atomic_sub_return arch_atomic_sub_return
396 : #endif
397 :
398 : #endif /* arch_atomic_sub_return_relaxed */
399 :
400 : #ifndef arch_atomic_fetch_sub_relaxed
401 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
402 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
403 : #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
404 : #else /* arch_atomic_fetch_sub_relaxed */
405 :
406 : #ifndef arch_atomic_fetch_sub_acquire
407 : static __always_inline int
408 : arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
409 : {
410 : int ret = arch_atomic_fetch_sub_relaxed(i, v);
411 : __atomic_acquire_fence();
412 : return ret;
413 : }
414 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
415 : #endif
416 :
417 : #ifndef arch_atomic_fetch_sub_release
418 : static __always_inline int
419 : arch_atomic_fetch_sub_release(int i, atomic_t *v)
420 : {
421 : __atomic_release_fence();
422 : return arch_atomic_fetch_sub_relaxed(i, v);
423 : }
424 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
425 : #endif
426 :
427 : #ifndef arch_atomic_fetch_sub
428 : static __always_inline int
429 : arch_atomic_fetch_sub(int i, atomic_t *v)
430 : {
431 : int ret;
432 : __atomic_pre_full_fence();
433 : ret = arch_atomic_fetch_sub_relaxed(i, v);
434 : __atomic_post_full_fence();
435 : return ret;
436 : }
437 : #define arch_atomic_fetch_sub arch_atomic_fetch_sub
438 : #endif
439 :
440 : #endif /* arch_atomic_fetch_sub_relaxed */
441 :
442 : #ifndef arch_atomic_inc
443 : static __always_inline void
444 : arch_atomic_inc(atomic_t *v)
445 : {
446 : arch_atomic_add(1, v);
447 : }
448 : #define arch_atomic_inc arch_atomic_inc
449 : #endif
450 :
451 : #ifndef arch_atomic_inc_return_relaxed
452 : #ifdef arch_atomic_inc_return
453 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return
454 : #define arch_atomic_inc_return_release arch_atomic_inc_return
455 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
456 : #endif /* arch_atomic_inc_return */
457 :
458 : #ifndef arch_atomic_inc_return
459 : static __always_inline int
460 : arch_atomic_inc_return(atomic_t *v)
461 : {
462 5 : return arch_atomic_add_return(1, v);
463 : }
464 : #define arch_atomic_inc_return arch_atomic_inc_return
465 : #endif
466 :
467 : #ifndef arch_atomic_inc_return_acquire
468 : static __always_inline int
469 : arch_atomic_inc_return_acquire(atomic_t *v)
470 : {
471 : return arch_atomic_add_return_acquire(1, v);
472 : }
473 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
474 : #endif
475 :
476 : #ifndef arch_atomic_inc_return_release
477 : static __always_inline int
478 : arch_atomic_inc_return_release(atomic_t *v)
479 : {
480 : return arch_atomic_add_return_release(1, v);
481 : }
482 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
483 : #endif
484 :
485 : #ifndef arch_atomic_inc_return_relaxed
486 : static __always_inline int
487 : arch_atomic_inc_return_relaxed(atomic_t *v)
488 : {
489 0 : return arch_atomic_add_return_relaxed(1, v);
490 : }
491 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
492 : #endif
493 :
494 : #else /* arch_atomic_inc_return_relaxed */
495 :
496 : #ifndef arch_atomic_inc_return_acquire
497 : static __always_inline int
498 : arch_atomic_inc_return_acquire(atomic_t *v)
499 : {
500 : int ret = arch_atomic_inc_return_relaxed(v);
501 : __atomic_acquire_fence();
502 : return ret;
503 : }
504 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
505 : #endif
506 :
507 : #ifndef arch_atomic_inc_return_release
508 : static __always_inline int
509 : arch_atomic_inc_return_release(atomic_t *v)
510 : {
511 : __atomic_release_fence();
512 : return arch_atomic_inc_return_relaxed(v);
513 : }
514 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
515 : #endif
516 :
517 : #ifndef arch_atomic_inc_return
518 : static __always_inline int
519 : arch_atomic_inc_return(atomic_t *v)
520 : {
521 : int ret;
522 : __atomic_pre_full_fence();
523 : ret = arch_atomic_inc_return_relaxed(v);
524 : __atomic_post_full_fence();
525 : return ret;
526 : }
527 : #define arch_atomic_inc_return arch_atomic_inc_return
528 : #endif
529 :
530 : #endif /* arch_atomic_inc_return_relaxed */
531 :
532 : #ifndef arch_atomic_fetch_inc_relaxed
533 : #ifdef arch_atomic_fetch_inc
534 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
535 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
536 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
537 : #endif /* arch_atomic_fetch_inc */
538 :
539 : #ifndef arch_atomic_fetch_inc
540 : static __always_inline int
541 : arch_atomic_fetch_inc(atomic_t *v)
542 : {
543 0 : return arch_atomic_fetch_add(1, v);
544 : }
545 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
546 : #endif
547 :
548 : #ifndef arch_atomic_fetch_inc_acquire
549 : static __always_inline int
550 : arch_atomic_fetch_inc_acquire(atomic_t *v)
551 : {
552 : return arch_atomic_fetch_add_acquire(1, v);
553 : }
554 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
555 : #endif
556 :
557 : #ifndef arch_atomic_fetch_inc_release
558 : static __always_inline int
559 : arch_atomic_fetch_inc_release(atomic_t *v)
560 : {
561 : return arch_atomic_fetch_add_release(1, v);
562 : }
563 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
564 : #endif
565 :
566 : #ifndef arch_atomic_fetch_inc_relaxed
567 : static __always_inline int
568 : arch_atomic_fetch_inc_relaxed(atomic_t *v)
569 : {
570 : return arch_atomic_fetch_add_relaxed(1, v);
571 : }
572 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
573 : #endif
574 :
575 : #else /* arch_atomic_fetch_inc_relaxed */
576 :
577 : #ifndef arch_atomic_fetch_inc_acquire
578 : static __always_inline int
579 : arch_atomic_fetch_inc_acquire(atomic_t *v)
580 : {
581 : int ret = arch_atomic_fetch_inc_relaxed(v);
582 : __atomic_acquire_fence();
583 : return ret;
584 : }
585 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
586 : #endif
587 :
588 : #ifndef arch_atomic_fetch_inc_release
589 : static __always_inline int
590 : arch_atomic_fetch_inc_release(atomic_t *v)
591 : {
592 : __atomic_release_fence();
593 : return arch_atomic_fetch_inc_relaxed(v);
594 : }
595 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
596 : #endif
597 :
598 : #ifndef arch_atomic_fetch_inc
599 : static __always_inline int
600 : arch_atomic_fetch_inc(atomic_t *v)
601 : {
602 : int ret;
603 : __atomic_pre_full_fence();
604 : ret = arch_atomic_fetch_inc_relaxed(v);
605 : __atomic_post_full_fence();
606 : return ret;
607 : }
608 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
609 : #endif
610 :
611 : #endif /* arch_atomic_fetch_inc_relaxed */
612 :
613 : #ifndef arch_atomic_dec
614 : static __always_inline void
615 : arch_atomic_dec(atomic_t *v)
616 : {
617 : arch_atomic_sub(1, v);
618 : }
619 : #define arch_atomic_dec arch_atomic_dec
620 : #endif
621 :
622 : #ifndef arch_atomic_dec_return_relaxed
623 : #ifdef arch_atomic_dec_return
624 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return
625 : #define arch_atomic_dec_return_release arch_atomic_dec_return
626 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
627 : #endif /* arch_atomic_dec_return */
628 :
629 : #ifndef arch_atomic_dec_return
630 : static __always_inline int
631 : arch_atomic_dec_return(atomic_t *v)
632 : {
633 0 : return arch_atomic_sub_return(1, v);
634 : }
635 : #define arch_atomic_dec_return arch_atomic_dec_return
636 : #endif
637 :
638 : #ifndef arch_atomic_dec_return_acquire
639 : static __always_inline int
640 : arch_atomic_dec_return_acquire(atomic_t *v)
641 : {
642 : return arch_atomic_sub_return_acquire(1, v);
643 : }
644 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
645 : #endif
646 :
647 : #ifndef arch_atomic_dec_return_release
648 : static __always_inline int
649 : arch_atomic_dec_return_release(atomic_t *v)
650 : {
651 : return arch_atomic_sub_return_release(1, v);
652 : }
653 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
654 : #endif
655 :
656 : #ifndef arch_atomic_dec_return_relaxed
657 : static __always_inline int
658 : arch_atomic_dec_return_relaxed(atomic_t *v)
659 : {
660 0 : return arch_atomic_sub_return_relaxed(1, v);
661 : }
662 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
663 : #endif
664 :
665 : #else /* arch_atomic_dec_return_relaxed */
666 :
667 : #ifndef arch_atomic_dec_return_acquire
668 : static __always_inline int
669 : arch_atomic_dec_return_acquire(atomic_t *v)
670 : {
671 : int ret = arch_atomic_dec_return_relaxed(v);
672 : __atomic_acquire_fence();
673 : return ret;
674 : }
675 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
676 : #endif
677 :
678 : #ifndef arch_atomic_dec_return_release
679 : static __always_inline int
680 : arch_atomic_dec_return_release(atomic_t *v)
681 : {
682 : __atomic_release_fence();
683 : return arch_atomic_dec_return_relaxed(v);
684 : }
685 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
686 : #endif
687 :
688 : #ifndef arch_atomic_dec_return
689 : static __always_inline int
690 : arch_atomic_dec_return(atomic_t *v)
691 : {
692 : int ret;
693 : __atomic_pre_full_fence();
694 : ret = arch_atomic_dec_return_relaxed(v);
695 : __atomic_post_full_fence();
696 : return ret;
697 : }
698 : #define arch_atomic_dec_return arch_atomic_dec_return
699 : #endif
700 :
701 : #endif /* arch_atomic_dec_return_relaxed */
702 :
703 : #ifndef arch_atomic_fetch_dec_relaxed
704 : #ifdef arch_atomic_fetch_dec
705 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
706 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
707 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
708 : #endif /* arch_atomic_fetch_dec */
709 :
710 : #ifndef arch_atomic_fetch_dec
711 : static __always_inline int
712 : arch_atomic_fetch_dec(atomic_t *v)
713 : {
714 : return arch_atomic_fetch_sub(1, v);
715 : }
716 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
717 : #endif
718 :
719 : #ifndef arch_atomic_fetch_dec_acquire
720 : static __always_inline int
721 : arch_atomic_fetch_dec_acquire(atomic_t *v)
722 : {
723 : return arch_atomic_fetch_sub_acquire(1, v);
724 : }
725 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
726 : #endif
727 :
728 : #ifndef arch_atomic_fetch_dec_release
729 : static __always_inline int
730 : arch_atomic_fetch_dec_release(atomic_t *v)
731 : {
732 : return arch_atomic_fetch_sub_release(1, v);
733 : }
734 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
735 : #endif
736 :
737 : #ifndef arch_atomic_fetch_dec_relaxed
738 : static __always_inline int
739 : arch_atomic_fetch_dec_relaxed(atomic_t *v)
740 : {
741 : return arch_atomic_fetch_sub_relaxed(1, v);
742 : }
743 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
744 : #endif
745 :
746 : #else /* arch_atomic_fetch_dec_relaxed */
747 :
748 : #ifndef arch_atomic_fetch_dec_acquire
749 : static __always_inline int
750 : arch_atomic_fetch_dec_acquire(atomic_t *v)
751 : {
752 : int ret = arch_atomic_fetch_dec_relaxed(v);
753 : __atomic_acquire_fence();
754 : return ret;
755 : }
756 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
757 : #endif
758 :
759 : #ifndef arch_atomic_fetch_dec_release
760 : static __always_inline int
761 : arch_atomic_fetch_dec_release(atomic_t *v)
762 : {
763 : __atomic_release_fence();
764 : return arch_atomic_fetch_dec_relaxed(v);
765 : }
766 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
767 : #endif
768 :
769 : #ifndef arch_atomic_fetch_dec
770 : static __always_inline int
771 : arch_atomic_fetch_dec(atomic_t *v)
772 : {
773 : int ret;
774 : __atomic_pre_full_fence();
775 : ret = arch_atomic_fetch_dec_relaxed(v);
776 : __atomic_post_full_fence();
777 : return ret;
778 : }
779 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
780 : #endif
781 :
782 : #endif /* arch_atomic_fetch_dec_relaxed */
783 :
784 : #ifndef arch_atomic_fetch_and_relaxed
785 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
786 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and
787 : #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
788 : #else /* arch_atomic_fetch_and_relaxed */
789 :
790 : #ifndef arch_atomic_fetch_and_acquire
791 : static __always_inline int
792 : arch_atomic_fetch_and_acquire(int i, atomic_t *v)
793 : {
794 : int ret = arch_atomic_fetch_and_relaxed(i, v);
795 : __atomic_acquire_fence();
796 : return ret;
797 : }
798 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
799 : #endif
800 :
801 : #ifndef arch_atomic_fetch_and_release
802 : static __always_inline int
803 : arch_atomic_fetch_and_release(int i, atomic_t *v)
804 : {
805 : __atomic_release_fence();
806 : return arch_atomic_fetch_and_relaxed(i, v);
807 : }
808 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
809 : #endif
810 :
811 : #ifndef arch_atomic_fetch_and
812 : static __always_inline int
813 : arch_atomic_fetch_and(int i, atomic_t *v)
814 : {
815 : int ret;
816 : __atomic_pre_full_fence();
817 : ret = arch_atomic_fetch_and_relaxed(i, v);
818 : __atomic_post_full_fence();
819 : return ret;
820 : }
821 : #define arch_atomic_fetch_and arch_atomic_fetch_and
822 : #endif
823 :
824 : #endif /* arch_atomic_fetch_and_relaxed */
825 :
826 : #ifndef arch_atomic_andnot
827 : static __always_inline void
828 : arch_atomic_andnot(int i, atomic_t *v)
829 : {
830 0 : arch_atomic_and(~i, v);
831 : }
832 : #define arch_atomic_andnot arch_atomic_andnot
833 : #endif
834 :
835 : #ifndef arch_atomic_fetch_andnot_relaxed
836 : #ifdef arch_atomic_fetch_andnot
837 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
838 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
839 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
840 : #endif /* arch_atomic_fetch_andnot */
841 :
842 : #ifndef arch_atomic_fetch_andnot
843 : static __always_inline int
844 : arch_atomic_fetch_andnot(int i, atomic_t *v)
845 : {
846 : return arch_atomic_fetch_and(~i, v);
847 : }
848 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
849 : #endif
850 :
851 : #ifndef arch_atomic_fetch_andnot_acquire
852 : static __always_inline int
853 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
854 : {
855 : return arch_atomic_fetch_and_acquire(~i, v);
856 : }
857 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
858 : #endif
859 :
860 : #ifndef arch_atomic_fetch_andnot_release
861 : static __always_inline int
862 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
863 : {
864 : return arch_atomic_fetch_and_release(~i, v);
865 : }
866 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
867 : #endif
868 :
869 : #ifndef arch_atomic_fetch_andnot_relaxed
870 : static __always_inline int
871 : arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
872 : {
873 : return arch_atomic_fetch_and_relaxed(~i, v);
874 : }
875 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
876 : #endif
877 :
878 : #else /* arch_atomic_fetch_andnot_relaxed */
879 :
880 : #ifndef arch_atomic_fetch_andnot_acquire
881 : static __always_inline int
882 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
883 : {
884 : int ret = arch_atomic_fetch_andnot_relaxed(i, v);
885 : __atomic_acquire_fence();
886 : return ret;
887 : }
888 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
889 : #endif
890 :
891 : #ifndef arch_atomic_fetch_andnot_release
892 : static __always_inline int
893 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
894 : {
895 : __atomic_release_fence();
896 : return arch_atomic_fetch_andnot_relaxed(i, v);
897 : }
898 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
899 : #endif
900 :
901 : #ifndef arch_atomic_fetch_andnot
902 : static __always_inline int
903 : arch_atomic_fetch_andnot(int i, atomic_t *v)
904 : {
905 : int ret;
906 : __atomic_pre_full_fence();
907 : ret = arch_atomic_fetch_andnot_relaxed(i, v);
908 : __atomic_post_full_fence();
909 : return ret;
910 : }
911 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
912 : #endif
913 :
914 : #endif /* arch_atomic_fetch_andnot_relaxed */
915 :
916 : #ifndef arch_atomic_fetch_or_relaxed
917 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
918 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or
919 : #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
920 : #else /* arch_atomic_fetch_or_relaxed */
921 :
922 : #ifndef arch_atomic_fetch_or_acquire
923 : static __always_inline int
924 : arch_atomic_fetch_or_acquire(int i, atomic_t *v)
925 : {
926 : int ret = arch_atomic_fetch_or_relaxed(i, v);
927 : __atomic_acquire_fence();
928 : return ret;
929 : }
930 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
931 : #endif
932 :
933 : #ifndef arch_atomic_fetch_or_release
934 : static __always_inline int
935 : arch_atomic_fetch_or_release(int i, atomic_t *v)
936 : {
937 : __atomic_release_fence();
938 : return arch_atomic_fetch_or_relaxed(i, v);
939 : }
940 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
941 : #endif
942 :
943 : #ifndef arch_atomic_fetch_or
944 : static __always_inline int
945 : arch_atomic_fetch_or(int i, atomic_t *v)
946 : {
947 : int ret;
948 : __atomic_pre_full_fence();
949 : ret = arch_atomic_fetch_or_relaxed(i, v);
950 : __atomic_post_full_fence();
951 : return ret;
952 : }
953 : #define arch_atomic_fetch_or arch_atomic_fetch_or
954 : #endif
955 :
956 : #endif /* arch_atomic_fetch_or_relaxed */
957 :
958 : #ifndef arch_atomic_fetch_xor_relaxed
959 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
960 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
961 : #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
962 : #else /* arch_atomic_fetch_xor_relaxed */
963 :
964 : #ifndef arch_atomic_fetch_xor_acquire
965 : static __always_inline int
966 : arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
967 : {
968 : int ret = arch_atomic_fetch_xor_relaxed(i, v);
969 : __atomic_acquire_fence();
970 : return ret;
971 : }
972 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
973 : #endif
974 :
975 : #ifndef arch_atomic_fetch_xor_release
976 : static __always_inline int
977 : arch_atomic_fetch_xor_release(int i, atomic_t *v)
978 : {
979 : __atomic_release_fence();
980 : return arch_atomic_fetch_xor_relaxed(i, v);
981 : }
982 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
983 : #endif
984 :
985 : #ifndef arch_atomic_fetch_xor
986 : static __always_inline int
987 : arch_atomic_fetch_xor(int i, atomic_t *v)
988 : {
989 : int ret;
990 : __atomic_pre_full_fence();
991 : ret = arch_atomic_fetch_xor_relaxed(i, v);
992 : __atomic_post_full_fence();
993 : return ret;
994 : }
995 : #define arch_atomic_fetch_xor arch_atomic_fetch_xor
996 : #endif
997 :
998 : #endif /* arch_atomic_fetch_xor_relaxed */
999 :
1000 : #ifndef arch_atomic_xchg_relaxed
1001 : #define arch_atomic_xchg_acquire arch_atomic_xchg
1002 : #define arch_atomic_xchg_release arch_atomic_xchg
1003 : #define arch_atomic_xchg_relaxed arch_atomic_xchg
1004 : #else /* arch_atomic_xchg_relaxed */
1005 :
1006 : #ifndef arch_atomic_xchg_acquire
1007 : static __always_inline int
1008 : arch_atomic_xchg_acquire(atomic_t *v, int i)
1009 : {
1010 : int ret = arch_atomic_xchg_relaxed(v, i);
1011 : __atomic_acquire_fence();
1012 : return ret;
1013 : }
1014 : #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
1015 : #endif
1016 :
1017 : #ifndef arch_atomic_xchg_release
1018 : static __always_inline int
1019 : arch_atomic_xchg_release(atomic_t *v, int i)
1020 : {
1021 : __atomic_release_fence();
1022 : return arch_atomic_xchg_relaxed(v, i);
1023 : }
1024 : #define arch_atomic_xchg_release arch_atomic_xchg_release
1025 : #endif
1026 :
1027 : #ifndef arch_atomic_xchg
1028 : static __always_inline int
1029 : arch_atomic_xchg(atomic_t *v, int i)
1030 : {
1031 : int ret;
1032 : __atomic_pre_full_fence();
1033 : ret = arch_atomic_xchg_relaxed(v, i);
1034 : __atomic_post_full_fence();
1035 : return ret;
1036 : }
1037 : #define arch_atomic_xchg arch_atomic_xchg
1038 : #endif
1039 :
1040 : #endif /* arch_atomic_xchg_relaxed */
1041 :
1042 : #ifndef arch_atomic_cmpxchg_relaxed
1043 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
1044 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
1045 : #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
1046 : #else /* arch_atomic_cmpxchg_relaxed */
1047 :
1048 : #ifndef arch_atomic_cmpxchg_acquire
1049 : static __always_inline int
1050 : arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1051 : {
1052 : int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1053 : __atomic_acquire_fence();
1054 : return ret;
1055 : }
1056 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
1057 : #endif
1058 :
1059 : #ifndef arch_atomic_cmpxchg_release
1060 : static __always_inline int
1061 : arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
1062 : {
1063 : __atomic_release_fence();
1064 : return arch_atomic_cmpxchg_relaxed(v, old, new);
1065 : }
1066 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
1067 : #endif
1068 :
1069 : #ifndef arch_atomic_cmpxchg
1070 : static __always_inline int
1071 : arch_atomic_cmpxchg(atomic_t *v, int old, int new)
1072 : {
1073 : int ret;
1074 : __atomic_pre_full_fence();
1075 : ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1076 : __atomic_post_full_fence();
1077 : return ret;
1078 : }
1079 : #define arch_atomic_cmpxchg arch_atomic_cmpxchg
1080 : #endif
1081 :
1082 : #endif /* arch_atomic_cmpxchg_relaxed */
1083 :
1084 : #ifndef arch_atomic_try_cmpxchg_relaxed
1085 : #ifdef arch_atomic_try_cmpxchg
1086 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
1087 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
1088 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
1089 : #endif /* arch_atomic_try_cmpxchg */
1090 :
1091 : #ifndef arch_atomic_try_cmpxchg
1092 : static __always_inline bool
1093 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1094 : {
1095 : int r, o = *old;
1096 : r = arch_atomic_cmpxchg(v, o, new);
1097 : if (unlikely(r != o))
1098 : *old = r;
1099 : return likely(r == o);
1100 : }
1101 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1102 : #endif
1103 :
1104 : #ifndef arch_atomic_try_cmpxchg_acquire
1105 : static __always_inline bool
1106 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1107 : {
1108 : int r, o = *old;
1109 : r = arch_atomic_cmpxchg_acquire(v, o, new);
1110 : if (unlikely(r != o))
1111 : *old = r;
1112 : return likely(r == o);
1113 : }
1114 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1115 : #endif
1116 :
1117 : #ifndef arch_atomic_try_cmpxchg_release
1118 : static __always_inline bool
1119 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1120 : {
1121 : int r, o = *old;
1122 : r = arch_atomic_cmpxchg_release(v, o, new);
1123 : if (unlikely(r != o))
1124 : *old = r;
1125 : return likely(r == o);
1126 : }
1127 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1128 : #endif
1129 :
1130 : #ifndef arch_atomic_try_cmpxchg_relaxed
1131 : static __always_inline bool
1132 : arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1133 : {
1134 : int r, o = *old;
1135 : r = arch_atomic_cmpxchg_relaxed(v, o, new);
1136 : if (unlikely(r != o))
1137 : *old = r;
1138 : return likely(r == o);
1139 : }
1140 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1141 : #endif
1142 :
1143 : #else /* arch_atomic_try_cmpxchg_relaxed */
1144 :
1145 : #ifndef arch_atomic_try_cmpxchg_acquire
1146 : static __always_inline bool
1147 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1148 : {
1149 : bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1150 : __atomic_acquire_fence();
1151 : return ret;
1152 : }
1153 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1154 : #endif
1155 :
1156 : #ifndef arch_atomic_try_cmpxchg_release
1157 : static __always_inline bool
1158 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1159 : {
1160 : __atomic_release_fence();
1161 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1162 : }
1163 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1164 : #endif
1165 :
1166 : #ifndef arch_atomic_try_cmpxchg
1167 : static __always_inline bool
1168 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1169 : {
1170 : bool ret;
1171 : __atomic_pre_full_fence();
1172 : ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1173 : __atomic_post_full_fence();
1174 : return ret;
1175 : }
1176 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1177 : #endif
1178 :
1179 : #endif /* arch_atomic_try_cmpxchg_relaxed */
1180 :
1181 : #ifndef arch_atomic_sub_and_test
1182 : /**
1183 : * arch_atomic_sub_and_test - subtract value from variable and test result
1184 : * @i: integer value to subtract
1185 : * @v: pointer of type atomic_t
1186 : *
1187 : * Atomically subtracts @i from @v and returns
1188 : * true if the result is zero, or false for all
1189 : * other cases.
1190 : */
1191 : static __always_inline bool
1192 : arch_atomic_sub_and_test(int i, atomic_t *v)
1193 : {
1194 : return arch_atomic_sub_return(i, v) == 0;
1195 : }
1196 : #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1197 : #endif
1198 :
1199 : #ifndef arch_atomic_dec_and_test
1200 : /**
1201 : * arch_atomic_dec_and_test - decrement and test
1202 : * @v: pointer of type atomic_t
1203 : *
1204 : * Atomically decrements @v by 1 and
1205 : * returns true if the result is 0, or false for all other
1206 : * cases.
1207 : */
1208 : static __always_inline bool
1209 : arch_atomic_dec_and_test(atomic_t *v)
1210 : {
1211 : return arch_atomic_dec_return(v) == 0;
1212 : }
1213 : #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1214 : #endif
1215 :
1216 : #ifndef arch_atomic_inc_and_test
1217 : /**
1218 : * arch_atomic_inc_and_test - increment and test
1219 : * @v: pointer of type atomic_t
1220 : *
1221 : * Atomically increments @v by 1
1222 : * and returns true if the result is zero, or false for all
1223 : * other cases.
1224 : */
1225 : static __always_inline bool
1226 : arch_atomic_inc_and_test(atomic_t *v)
1227 : {
1228 : return arch_atomic_inc_return(v) == 0;
1229 : }
1230 : #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1231 : #endif
1232 :
1233 : #ifndef arch_atomic_add_negative_relaxed
1234 : #ifdef arch_atomic_add_negative
1235 : #define arch_atomic_add_negative_acquire arch_atomic_add_negative
1236 : #define arch_atomic_add_negative_release arch_atomic_add_negative
1237 : #define arch_atomic_add_negative_relaxed arch_atomic_add_negative
1238 : #endif /* arch_atomic_add_negative */
1239 :
1240 : #ifndef arch_atomic_add_negative
1241 : /**
1242 : * arch_atomic_add_negative - Add and test if negative
1243 : * @i: integer value to add
1244 : * @v: pointer of type atomic_t
1245 : *
1246 : * Atomically adds @i to @v and returns true if the result is negative,
1247 : * or false when the result is greater than or equal to zero.
1248 : */
1249 : static __always_inline bool
1250 : arch_atomic_add_negative(int i, atomic_t *v)
1251 : {
1252 : return arch_atomic_add_return(i, v) < 0;
1253 : }
1254 : #define arch_atomic_add_negative arch_atomic_add_negative
1255 : #endif
1256 :
1257 : #ifndef arch_atomic_add_negative_acquire
1258 : /**
1259 : * arch_atomic_add_negative_acquire - Add and test if negative
1260 : * @i: integer value to add
1261 : * @v: pointer of type atomic_t
1262 : *
1263 : * Atomically adds @i to @v and returns true if the result is negative,
1264 : * or false when the result is greater than or equal to zero.
1265 : */
1266 : static __always_inline bool
1267 : arch_atomic_add_negative_acquire(int i, atomic_t *v)
1268 : {
1269 : return arch_atomic_add_return_acquire(i, v) < 0;
1270 : }
1271 : #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
1272 : #endif
1273 :
1274 : #ifndef arch_atomic_add_negative_release
1275 : /**
1276 : * arch_atomic_add_negative_release - Add and test if negative
1277 : * @i: integer value to add
1278 : * @v: pointer of type atomic_t
1279 : *
1280 : * Atomically adds @i to @v and returns true if the result is negative,
1281 : * or false when the result is greater than or equal to zero.
1282 : */
1283 : static __always_inline bool
1284 : arch_atomic_add_negative_release(int i, atomic_t *v)
1285 : {
1286 : return arch_atomic_add_return_release(i, v) < 0;
1287 : }
1288 : #define arch_atomic_add_negative_release arch_atomic_add_negative_release
1289 : #endif
1290 :
1291 : #ifndef arch_atomic_add_negative_relaxed
1292 : /**
1293 : * arch_atomic_add_negative_relaxed - Add and test if negative
1294 : * @i: integer value to add
1295 : * @v: pointer of type atomic_t
1296 : *
1297 : * Atomically adds @i to @v and returns true if the result is negative,
1298 : * or false when the result is greater than or equal to zero.
1299 : */
1300 : static __always_inline bool
1301 : arch_atomic_add_negative_relaxed(int i, atomic_t *v)
1302 : {
1303 : return arch_atomic_add_return_relaxed(i, v) < 0;
1304 : }
1305 : #define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed
1306 : #endif
1307 :
1308 : #else /* arch_atomic_add_negative_relaxed */
1309 :
1310 : #ifndef arch_atomic_add_negative_acquire
1311 : static __always_inline bool
1312 : arch_atomic_add_negative_acquire(int i, atomic_t *v)
1313 : {
1314 : bool ret = arch_atomic_add_negative_relaxed(i, v);
1315 : __atomic_acquire_fence();
1316 : return ret;
1317 : }
1318 : #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire
1319 : #endif
1320 :
1321 : #ifndef arch_atomic_add_negative_release
1322 : static __always_inline bool
1323 : arch_atomic_add_negative_release(int i, atomic_t *v)
1324 : {
1325 : __atomic_release_fence();
1326 : return arch_atomic_add_negative_relaxed(i, v);
1327 : }
1328 : #define arch_atomic_add_negative_release arch_atomic_add_negative_release
1329 : #endif
1330 :
1331 : #ifndef arch_atomic_add_negative
1332 : static __always_inline bool
1333 : arch_atomic_add_negative(int i, atomic_t *v)
1334 : {
1335 : bool ret;
1336 : __atomic_pre_full_fence();
1337 : ret = arch_atomic_add_negative_relaxed(i, v);
1338 : __atomic_post_full_fence();
1339 : return ret;
1340 : }
1341 : #define arch_atomic_add_negative arch_atomic_add_negative
1342 : #endif
1343 :
1344 : #endif /* arch_atomic_add_negative_relaxed */
1345 :
1346 : #ifndef arch_atomic_fetch_add_unless
1347 : /**
1348 : * arch_atomic_fetch_add_unless - add unless the number is already a given value
1349 : * @v: pointer of type atomic_t
1350 : * @a: the amount to add to v...
1351 : * @u: ...unless v is equal to u.
1352 : *
1353 : * Atomically adds @a to @v, so long as @v was not already @u.
1354 : * Returns original value of @v
1355 : */
1356 : static __always_inline int
1357 : arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1358 : {
1359 1022 : int c = arch_atomic_read(v);
1360 :
1361 : do {
1362 1022 : if (unlikely(c == u))
1363 : break;
1364 1912 : } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1365 :
1366 : return c;
1367 : }
1368 : #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1369 : #endif
1370 :
1371 : #ifndef arch_atomic_add_unless
1372 : /**
1373 : * arch_atomic_add_unless - add unless the number is already a given value
1374 : * @v: pointer of type atomic_t
1375 : * @a: the amount to add to v...
1376 : * @u: ...unless v is equal to u.
1377 : *
1378 : * Atomically adds @a to @v, if @v was not already @u.
1379 : * Returns true if the addition was done.
1380 : */
1381 : static __always_inline bool
1382 : arch_atomic_add_unless(atomic_t *v, int a, int u)
1383 : {
1384 1022 : return arch_atomic_fetch_add_unless(v, a, u) != u;
1385 : }
1386 : #define arch_atomic_add_unless arch_atomic_add_unless
1387 : #endif
1388 :
1389 : #ifndef arch_atomic_inc_not_zero
1390 : /**
1391 : * arch_atomic_inc_not_zero - increment unless the number is zero
1392 : * @v: pointer of type atomic_t
1393 : *
1394 : * Atomically increments @v by 1, if @v is non-zero.
1395 : * Returns true if the increment was done.
1396 : */
1397 : static __always_inline bool
1398 : arch_atomic_inc_not_zero(atomic_t *v)
1399 : {
1400 0 : return arch_atomic_add_unless(v, 1, 0);
1401 : }
1402 : #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1403 : #endif
1404 :
1405 : #ifndef arch_atomic_inc_unless_negative
1406 : static __always_inline bool
1407 : arch_atomic_inc_unless_negative(atomic_t *v)
1408 : {
1409 0 : int c = arch_atomic_read(v);
1410 :
1411 : do {
1412 0 : if (unlikely(c < 0))
1413 : return false;
1414 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1415 :
1416 : return true;
1417 : }
1418 : #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1419 : #endif
1420 :
1421 : #ifndef arch_atomic_dec_unless_positive
1422 : static __always_inline bool
1423 : arch_atomic_dec_unless_positive(atomic_t *v)
1424 : {
1425 0 : int c = arch_atomic_read(v);
1426 :
1427 : do {
1428 0 : if (unlikely(c > 0))
1429 : return false;
1430 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1431 :
1432 : return true;
1433 : }
1434 : #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1435 : #endif
1436 :
1437 : #ifndef arch_atomic_dec_if_positive
1438 : static __always_inline int
1439 : arch_atomic_dec_if_positive(atomic_t *v)
1440 : {
1441 0 : int dec, c = arch_atomic_read(v);
1442 :
1443 : do {
1444 0 : dec = c - 1;
1445 0 : if (unlikely(dec < 0))
1446 : break;
1447 0 : } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1448 :
1449 : return dec;
1450 : }
1451 : #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1452 : #endif
1453 :
1454 : #ifdef CONFIG_GENERIC_ATOMIC64
1455 : #include <asm-generic/atomic64.h>
1456 : #endif
1457 :
1458 : #ifndef arch_atomic64_read_acquire
1459 : static __always_inline s64
1460 : arch_atomic64_read_acquire(const atomic64_t *v)
1461 : {
1462 : s64 ret;
1463 :
1464 : if (__native_word(atomic64_t)) {
1465 : ret = smp_load_acquire(&(v)->counter);
1466 : } else {
1467 : ret = arch_atomic64_read(v);
1468 : __atomic_acquire_fence();
1469 : }
1470 :
1471 : return ret;
1472 : }
1473 : #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1474 : #endif
1475 :
1476 : #ifndef arch_atomic64_set_release
1477 : static __always_inline void
1478 : arch_atomic64_set_release(atomic64_t *v, s64 i)
1479 : {
1480 : if (__native_word(atomic64_t)) {
1481 : smp_store_release(&(v)->counter, i);
1482 : } else {
1483 : __atomic_release_fence();
1484 : arch_atomic64_set(v, i);
1485 : }
1486 : }
1487 : #define arch_atomic64_set_release arch_atomic64_set_release
1488 : #endif
1489 :
1490 : #ifndef arch_atomic64_add_return_relaxed
1491 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1492 : #define arch_atomic64_add_return_release arch_atomic64_add_return
1493 : #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1494 : #else /* arch_atomic64_add_return_relaxed */
1495 :
1496 : #ifndef arch_atomic64_add_return_acquire
1497 : static __always_inline s64
1498 : arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1499 : {
1500 : s64 ret = arch_atomic64_add_return_relaxed(i, v);
1501 : __atomic_acquire_fence();
1502 : return ret;
1503 : }
1504 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1505 : #endif
1506 :
1507 : #ifndef arch_atomic64_add_return_release
1508 : static __always_inline s64
1509 : arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1510 : {
1511 : __atomic_release_fence();
1512 : return arch_atomic64_add_return_relaxed(i, v);
1513 : }
1514 : #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1515 : #endif
1516 :
1517 : #ifndef arch_atomic64_add_return
1518 : static __always_inline s64
1519 : arch_atomic64_add_return(s64 i, atomic64_t *v)
1520 : {
1521 : s64 ret;
1522 : __atomic_pre_full_fence();
1523 : ret = arch_atomic64_add_return_relaxed(i, v);
1524 : __atomic_post_full_fence();
1525 : return ret;
1526 : }
1527 : #define arch_atomic64_add_return arch_atomic64_add_return
1528 : #endif
1529 :
1530 : #endif /* arch_atomic64_add_return_relaxed */
1531 :
1532 : #ifndef arch_atomic64_fetch_add_relaxed
1533 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1534 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1535 : #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1536 : #else /* arch_atomic64_fetch_add_relaxed */
1537 :
1538 : #ifndef arch_atomic64_fetch_add_acquire
1539 : static __always_inline s64
1540 : arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1541 : {
1542 : s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1543 : __atomic_acquire_fence();
1544 : return ret;
1545 : }
1546 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1547 : #endif
1548 :
1549 : #ifndef arch_atomic64_fetch_add_release
1550 : static __always_inline s64
1551 : arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1552 : {
1553 : __atomic_release_fence();
1554 : return arch_atomic64_fetch_add_relaxed(i, v);
1555 : }
1556 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1557 : #endif
1558 :
1559 : #ifndef arch_atomic64_fetch_add
1560 : static __always_inline s64
1561 : arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1562 : {
1563 : s64 ret;
1564 : __atomic_pre_full_fence();
1565 : ret = arch_atomic64_fetch_add_relaxed(i, v);
1566 : __atomic_post_full_fence();
1567 : return ret;
1568 : }
1569 : #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1570 : #endif
1571 :
1572 : #endif /* arch_atomic64_fetch_add_relaxed */
1573 :
1574 : #ifndef arch_atomic64_sub_return_relaxed
1575 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1576 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1577 : #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1578 : #else /* arch_atomic64_sub_return_relaxed */
1579 :
1580 : #ifndef arch_atomic64_sub_return_acquire
1581 : static __always_inline s64
1582 : arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1583 : {
1584 : s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1585 : __atomic_acquire_fence();
1586 : return ret;
1587 : }
1588 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1589 : #endif
1590 :
1591 : #ifndef arch_atomic64_sub_return_release
1592 : static __always_inline s64
1593 : arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1594 : {
1595 : __atomic_release_fence();
1596 : return arch_atomic64_sub_return_relaxed(i, v);
1597 : }
1598 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1599 : #endif
1600 :
1601 : #ifndef arch_atomic64_sub_return
1602 : static __always_inline s64
1603 : arch_atomic64_sub_return(s64 i, atomic64_t *v)
1604 : {
1605 : s64 ret;
1606 : __atomic_pre_full_fence();
1607 : ret = arch_atomic64_sub_return_relaxed(i, v);
1608 : __atomic_post_full_fence();
1609 : return ret;
1610 : }
1611 : #define arch_atomic64_sub_return arch_atomic64_sub_return
1612 : #endif
1613 :
1614 : #endif /* arch_atomic64_sub_return_relaxed */
1615 :
1616 : #ifndef arch_atomic64_fetch_sub_relaxed
1617 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1618 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1619 : #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1620 : #else /* arch_atomic64_fetch_sub_relaxed */
1621 :
1622 : #ifndef arch_atomic64_fetch_sub_acquire
1623 : static __always_inline s64
1624 : arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1625 : {
1626 : s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1627 : __atomic_acquire_fence();
1628 : return ret;
1629 : }
1630 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1631 : #endif
1632 :
1633 : #ifndef arch_atomic64_fetch_sub_release
1634 : static __always_inline s64
1635 : arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1636 : {
1637 : __atomic_release_fence();
1638 : return arch_atomic64_fetch_sub_relaxed(i, v);
1639 : }
1640 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1641 : #endif
1642 :
1643 : #ifndef arch_atomic64_fetch_sub
1644 : static __always_inline s64
1645 : arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1646 : {
1647 : s64 ret;
1648 : __atomic_pre_full_fence();
1649 : ret = arch_atomic64_fetch_sub_relaxed(i, v);
1650 : __atomic_post_full_fence();
1651 : return ret;
1652 : }
1653 : #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1654 : #endif
1655 :
1656 : #endif /* arch_atomic64_fetch_sub_relaxed */
1657 :
1658 : #ifndef arch_atomic64_inc
1659 : static __always_inline void
1660 : arch_atomic64_inc(atomic64_t *v)
1661 : {
1662 : arch_atomic64_add(1, v);
1663 : }
1664 : #define arch_atomic64_inc arch_atomic64_inc
1665 : #endif
1666 :
1667 : #ifndef arch_atomic64_inc_return_relaxed
1668 : #ifdef arch_atomic64_inc_return
1669 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1670 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1671 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1672 : #endif /* arch_atomic64_inc_return */
1673 :
1674 : #ifndef arch_atomic64_inc_return
1675 : static __always_inline s64
1676 : arch_atomic64_inc_return(atomic64_t *v)
1677 : {
1678 0 : return arch_atomic64_add_return(1, v);
1679 : }
1680 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1681 : #endif
1682 :
1683 : #ifndef arch_atomic64_inc_return_acquire
1684 : static __always_inline s64
1685 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1686 : {
1687 : return arch_atomic64_add_return_acquire(1, v);
1688 : }
1689 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1690 : #endif
1691 :
1692 : #ifndef arch_atomic64_inc_return_release
1693 : static __always_inline s64
1694 : arch_atomic64_inc_return_release(atomic64_t *v)
1695 : {
1696 : return arch_atomic64_add_return_release(1, v);
1697 : }
1698 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1699 : #endif
1700 :
1701 : #ifndef arch_atomic64_inc_return_relaxed
1702 : static __always_inline s64
1703 : arch_atomic64_inc_return_relaxed(atomic64_t *v)
1704 : {
1705 10 : return arch_atomic64_add_return_relaxed(1, v);
1706 : }
1707 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1708 : #endif
1709 :
1710 : #else /* arch_atomic64_inc_return_relaxed */
1711 :
1712 : #ifndef arch_atomic64_inc_return_acquire
1713 : static __always_inline s64
1714 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1715 : {
1716 : s64 ret = arch_atomic64_inc_return_relaxed(v);
1717 : __atomic_acquire_fence();
1718 : return ret;
1719 : }
1720 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1721 : #endif
1722 :
1723 : #ifndef arch_atomic64_inc_return_release
1724 : static __always_inline s64
1725 : arch_atomic64_inc_return_release(atomic64_t *v)
1726 : {
1727 : __atomic_release_fence();
1728 : return arch_atomic64_inc_return_relaxed(v);
1729 : }
1730 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1731 : #endif
1732 :
1733 : #ifndef arch_atomic64_inc_return
1734 : static __always_inline s64
1735 : arch_atomic64_inc_return(atomic64_t *v)
1736 : {
1737 : s64 ret;
1738 : __atomic_pre_full_fence();
1739 : ret = arch_atomic64_inc_return_relaxed(v);
1740 : __atomic_post_full_fence();
1741 : return ret;
1742 : }
1743 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1744 : #endif
1745 :
1746 : #endif /* arch_atomic64_inc_return_relaxed */
1747 :
1748 : #ifndef arch_atomic64_fetch_inc_relaxed
1749 : #ifdef arch_atomic64_fetch_inc
1750 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1751 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1752 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1753 : #endif /* arch_atomic64_fetch_inc */
1754 :
1755 : #ifndef arch_atomic64_fetch_inc
1756 : static __always_inline s64
1757 : arch_atomic64_fetch_inc(atomic64_t *v)
1758 : {
1759 : return arch_atomic64_fetch_add(1, v);
1760 : }
1761 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1762 : #endif
1763 :
1764 : #ifndef arch_atomic64_fetch_inc_acquire
1765 : static __always_inline s64
1766 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1767 : {
1768 : return arch_atomic64_fetch_add_acquire(1, v);
1769 : }
1770 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1771 : #endif
1772 :
1773 : #ifndef arch_atomic64_fetch_inc_release
1774 : static __always_inline s64
1775 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1776 : {
1777 : return arch_atomic64_fetch_add_release(1, v);
1778 : }
1779 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1780 : #endif
1781 :
1782 : #ifndef arch_atomic64_fetch_inc_relaxed
1783 : static __always_inline s64
1784 : arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1785 : {
1786 : return arch_atomic64_fetch_add_relaxed(1, v);
1787 : }
1788 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1789 : #endif
1790 :
1791 : #else /* arch_atomic64_fetch_inc_relaxed */
1792 :
1793 : #ifndef arch_atomic64_fetch_inc_acquire
1794 : static __always_inline s64
1795 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1796 : {
1797 : s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1798 : __atomic_acquire_fence();
1799 : return ret;
1800 : }
1801 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1802 : #endif
1803 :
1804 : #ifndef arch_atomic64_fetch_inc_release
1805 : static __always_inline s64
1806 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1807 : {
1808 : __atomic_release_fence();
1809 : return arch_atomic64_fetch_inc_relaxed(v);
1810 : }
1811 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1812 : #endif
1813 :
1814 : #ifndef arch_atomic64_fetch_inc
1815 : static __always_inline s64
1816 : arch_atomic64_fetch_inc(atomic64_t *v)
1817 : {
1818 : s64 ret;
1819 : __atomic_pre_full_fence();
1820 : ret = arch_atomic64_fetch_inc_relaxed(v);
1821 : __atomic_post_full_fence();
1822 : return ret;
1823 : }
1824 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1825 : #endif
1826 :
1827 : #endif /* arch_atomic64_fetch_inc_relaxed */
1828 :
1829 : #ifndef arch_atomic64_dec
1830 : static __always_inline void
1831 : arch_atomic64_dec(atomic64_t *v)
1832 : {
1833 : arch_atomic64_sub(1, v);
1834 : }
1835 : #define arch_atomic64_dec arch_atomic64_dec
1836 : #endif
1837 :
1838 : #ifndef arch_atomic64_dec_return_relaxed
1839 : #ifdef arch_atomic64_dec_return
1840 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1841 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1842 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1843 : #endif /* arch_atomic64_dec_return */
1844 :
1845 : #ifndef arch_atomic64_dec_return
1846 : static __always_inline s64
1847 : arch_atomic64_dec_return(atomic64_t *v)
1848 : {
1849 : return arch_atomic64_sub_return(1, v);
1850 : }
1851 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1852 : #endif
1853 :
1854 : #ifndef arch_atomic64_dec_return_acquire
1855 : static __always_inline s64
1856 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1857 : {
1858 : return arch_atomic64_sub_return_acquire(1, v);
1859 : }
1860 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1861 : #endif
1862 :
1863 : #ifndef arch_atomic64_dec_return_release
1864 : static __always_inline s64
1865 : arch_atomic64_dec_return_release(atomic64_t *v)
1866 : {
1867 : return arch_atomic64_sub_return_release(1, v);
1868 : }
1869 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1870 : #endif
1871 :
1872 : #ifndef arch_atomic64_dec_return_relaxed
1873 : static __always_inline s64
1874 : arch_atomic64_dec_return_relaxed(atomic64_t *v)
1875 : {
1876 : return arch_atomic64_sub_return_relaxed(1, v);
1877 : }
1878 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1879 : #endif
1880 :
1881 : #else /* arch_atomic64_dec_return_relaxed */
1882 :
1883 : #ifndef arch_atomic64_dec_return_acquire
1884 : static __always_inline s64
1885 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1886 : {
1887 : s64 ret = arch_atomic64_dec_return_relaxed(v);
1888 : __atomic_acquire_fence();
1889 : return ret;
1890 : }
1891 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1892 : #endif
1893 :
1894 : #ifndef arch_atomic64_dec_return_release
1895 : static __always_inline s64
1896 : arch_atomic64_dec_return_release(atomic64_t *v)
1897 : {
1898 : __atomic_release_fence();
1899 : return arch_atomic64_dec_return_relaxed(v);
1900 : }
1901 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1902 : #endif
1903 :
1904 : #ifndef arch_atomic64_dec_return
1905 : static __always_inline s64
1906 : arch_atomic64_dec_return(atomic64_t *v)
1907 : {
1908 : s64 ret;
1909 : __atomic_pre_full_fence();
1910 : ret = arch_atomic64_dec_return_relaxed(v);
1911 : __atomic_post_full_fence();
1912 : return ret;
1913 : }
1914 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1915 : #endif
1916 :
1917 : #endif /* arch_atomic64_dec_return_relaxed */
1918 :
1919 : #ifndef arch_atomic64_fetch_dec_relaxed
1920 : #ifdef arch_atomic64_fetch_dec
1921 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1922 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1923 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1924 : #endif /* arch_atomic64_fetch_dec */
1925 :
1926 : #ifndef arch_atomic64_fetch_dec
1927 : static __always_inline s64
1928 : arch_atomic64_fetch_dec(atomic64_t *v)
1929 : {
1930 : return arch_atomic64_fetch_sub(1, v);
1931 : }
1932 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1933 : #endif
1934 :
1935 : #ifndef arch_atomic64_fetch_dec_acquire
1936 : static __always_inline s64
1937 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1938 : {
1939 : return arch_atomic64_fetch_sub_acquire(1, v);
1940 : }
1941 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1942 : #endif
1943 :
1944 : #ifndef arch_atomic64_fetch_dec_release
1945 : static __always_inline s64
1946 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1947 : {
1948 : return arch_atomic64_fetch_sub_release(1, v);
1949 : }
1950 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1951 : #endif
1952 :
1953 : #ifndef arch_atomic64_fetch_dec_relaxed
1954 : static __always_inline s64
1955 : arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1956 : {
1957 : return arch_atomic64_fetch_sub_relaxed(1, v);
1958 : }
1959 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1960 : #endif
1961 :
1962 : #else /* arch_atomic64_fetch_dec_relaxed */
1963 :
1964 : #ifndef arch_atomic64_fetch_dec_acquire
1965 : static __always_inline s64
1966 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1967 : {
1968 : s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1969 : __atomic_acquire_fence();
1970 : return ret;
1971 : }
1972 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1973 : #endif
1974 :
1975 : #ifndef arch_atomic64_fetch_dec_release
1976 : static __always_inline s64
1977 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1978 : {
1979 : __atomic_release_fence();
1980 : return arch_atomic64_fetch_dec_relaxed(v);
1981 : }
1982 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1983 : #endif
1984 :
1985 : #ifndef arch_atomic64_fetch_dec
1986 : static __always_inline s64
1987 : arch_atomic64_fetch_dec(atomic64_t *v)
1988 : {
1989 : s64 ret;
1990 : __atomic_pre_full_fence();
1991 : ret = arch_atomic64_fetch_dec_relaxed(v);
1992 : __atomic_post_full_fence();
1993 : return ret;
1994 : }
1995 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1996 : #endif
1997 :
1998 : #endif /* arch_atomic64_fetch_dec_relaxed */
1999 :
2000 : #ifndef arch_atomic64_fetch_and_relaxed
2001 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
2002 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
2003 : #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
2004 : #else /* arch_atomic64_fetch_and_relaxed */
2005 :
2006 : #ifndef arch_atomic64_fetch_and_acquire
2007 : static __always_inline s64
2008 : arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
2009 : {
2010 : s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
2011 : __atomic_acquire_fence();
2012 : return ret;
2013 : }
2014 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
2015 : #endif
2016 :
2017 : #ifndef arch_atomic64_fetch_and_release
2018 : static __always_inline s64
2019 : arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
2020 : {
2021 : __atomic_release_fence();
2022 : return arch_atomic64_fetch_and_relaxed(i, v);
2023 : }
2024 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
2025 : #endif
2026 :
2027 : #ifndef arch_atomic64_fetch_and
2028 : static __always_inline s64
2029 : arch_atomic64_fetch_and(s64 i, atomic64_t *v)
2030 : {
2031 : s64 ret;
2032 : __atomic_pre_full_fence();
2033 : ret = arch_atomic64_fetch_and_relaxed(i, v);
2034 : __atomic_post_full_fence();
2035 : return ret;
2036 : }
2037 : #define arch_atomic64_fetch_and arch_atomic64_fetch_and
2038 : #endif
2039 :
2040 : #endif /* arch_atomic64_fetch_and_relaxed */
2041 :
2042 : #ifndef arch_atomic64_andnot
2043 : static __always_inline void
2044 : arch_atomic64_andnot(s64 i, atomic64_t *v)
2045 : {
2046 0 : arch_atomic64_and(~i, v);
2047 : }
2048 : #define arch_atomic64_andnot arch_atomic64_andnot
2049 : #endif
2050 :
2051 : #ifndef arch_atomic64_fetch_andnot_relaxed
2052 : #ifdef arch_atomic64_fetch_andnot
2053 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
2054 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
2055 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
2056 : #endif /* arch_atomic64_fetch_andnot */
2057 :
2058 : #ifndef arch_atomic64_fetch_andnot
2059 : static __always_inline s64
2060 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2061 : {
2062 : return arch_atomic64_fetch_and(~i, v);
2063 : }
2064 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
2065 : #endif
2066 :
2067 : #ifndef arch_atomic64_fetch_andnot_acquire
2068 : static __always_inline s64
2069 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2070 : {
2071 : return arch_atomic64_fetch_and_acquire(~i, v);
2072 : }
2073 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
2074 : #endif
2075 :
2076 : #ifndef arch_atomic64_fetch_andnot_release
2077 : static __always_inline s64
2078 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2079 : {
2080 : return arch_atomic64_fetch_and_release(~i, v);
2081 : }
2082 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
2083 : #endif
2084 :
2085 : #ifndef arch_atomic64_fetch_andnot_relaxed
2086 : static __always_inline s64
2087 : arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
2088 : {
2089 : return arch_atomic64_fetch_and_relaxed(~i, v);
2090 : }
2091 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
2092 : #endif
2093 :
2094 : #else /* arch_atomic64_fetch_andnot_relaxed */
2095 :
2096 : #ifndef arch_atomic64_fetch_andnot_acquire
2097 : static __always_inline s64
2098 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
2099 : {
2100 : s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2101 : __atomic_acquire_fence();
2102 : return ret;
2103 : }
2104 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
2105 : #endif
2106 :
2107 : #ifndef arch_atomic64_fetch_andnot_release
2108 : static __always_inline s64
2109 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
2110 : {
2111 : __atomic_release_fence();
2112 : return arch_atomic64_fetch_andnot_relaxed(i, v);
2113 : }
2114 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
2115 : #endif
2116 :
2117 : #ifndef arch_atomic64_fetch_andnot
2118 : static __always_inline s64
2119 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2120 : {
2121 : s64 ret;
2122 : __atomic_pre_full_fence();
2123 : ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2124 : __atomic_post_full_fence();
2125 : return ret;
2126 : }
2127 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
2128 : #endif
2129 :
2130 : #endif /* arch_atomic64_fetch_andnot_relaxed */
2131 :
2132 : #ifndef arch_atomic64_fetch_or_relaxed
2133 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
2134 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
2135 : #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
2136 : #else /* arch_atomic64_fetch_or_relaxed */
2137 :
2138 : #ifndef arch_atomic64_fetch_or_acquire
2139 : static __always_inline s64
2140 : arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2141 : {
2142 : s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
2143 : __atomic_acquire_fence();
2144 : return ret;
2145 : }
2146 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
2147 : #endif
2148 :
2149 : #ifndef arch_atomic64_fetch_or_release
2150 : static __always_inline s64
2151 : arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
2152 : {
2153 : __atomic_release_fence();
2154 : return arch_atomic64_fetch_or_relaxed(i, v);
2155 : }
2156 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
2157 : #endif
2158 :
2159 : #ifndef arch_atomic64_fetch_or
2160 : static __always_inline s64
2161 : arch_atomic64_fetch_or(s64 i, atomic64_t *v)
2162 : {
2163 : s64 ret;
2164 : __atomic_pre_full_fence();
2165 : ret = arch_atomic64_fetch_or_relaxed(i, v);
2166 : __atomic_post_full_fence();
2167 : return ret;
2168 : }
2169 : #define arch_atomic64_fetch_or arch_atomic64_fetch_or
2170 : #endif
2171 :
2172 : #endif /* arch_atomic64_fetch_or_relaxed */
2173 :
2174 : #ifndef arch_atomic64_fetch_xor_relaxed
2175 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
2176 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
2177 : #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
2178 : #else /* arch_atomic64_fetch_xor_relaxed */
2179 :
2180 : #ifndef arch_atomic64_fetch_xor_acquire
2181 : static __always_inline s64
2182 : arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2183 : {
2184 : s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
2185 : __atomic_acquire_fence();
2186 : return ret;
2187 : }
2188 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
2189 : #endif
2190 :
2191 : #ifndef arch_atomic64_fetch_xor_release
2192 : static __always_inline s64
2193 : arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2194 : {
2195 : __atomic_release_fence();
2196 : return arch_atomic64_fetch_xor_relaxed(i, v);
2197 : }
2198 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
2199 : #endif
2200 :
2201 : #ifndef arch_atomic64_fetch_xor
2202 : static __always_inline s64
2203 : arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
2204 : {
2205 : s64 ret;
2206 : __atomic_pre_full_fence();
2207 : ret = arch_atomic64_fetch_xor_relaxed(i, v);
2208 : __atomic_post_full_fence();
2209 : return ret;
2210 : }
2211 : #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
2212 : #endif
2213 :
2214 : #endif /* arch_atomic64_fetch_xor_relaxed */
2215 :
2216 : #ifndef arch_atomic64_xchg_relaxed
2217 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg
2218 : #define arch_atomic64_xchg_release arch_atomic64_xchg
2219 : #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2220 : #else /* arch_atomic64_xchg_relaxed */
2221 :
2222 : #ifndef arch_atomic64_xchg_acquire
2223 : static __always_inline s64
2224 : arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2225 : {
2226 : s64 ret = arch_atomic64_xchg_relaxed(v, i);
2227 : __atomic_acquire_fence();
2228 : return ret;
2229 : }
2230 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2231 : #endif
2232 :
2233 : #ifndef arch_atomic64_xchg_release
2234 : static __always_inline s64
2235 : arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2236 : {
2237 : __atomic_release_fence();
2238 : return arch_atomic64_xchg_relaxed(v, i);
2239 : }
2240 : #define arch_atomic64_xchg_release arch_atomic64_xchg_release
2241 : #endif
2242 :
2243 : #ifndef arch_atomic64_xchg
2244 : static __always_inline s64
2245 : arch_atomic64_xchg(atomic64_t *v, s64 i)
2246 : {
2247 : s64 ret;
2248 : __atomic_pre_full_fence();
2249 : ret = arch_atomic64_xchg_relaxed(v, i);
2250 : __atomic_post_full_fence();
2251 : return ret;
2252 : }
2253 : #define arch_atomic64_xchg arch_atomic64_xchg
2254 : #endif
2255 :
2256 : #endif /* arch_atomic64_xchg_relaxed */
2257 :
2258 : #ifndef arch_atomic64_cmpxchg_relaxed
2259 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2260 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2261 : #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2262 : #else /* arch_atomic64_cmpxchg_relaxed */
2263 :
2264 : #ifndef arch_atomic64_cmpxchg_acquire
2265 : static __always_inline s64
2266 : arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2267 : {
2268 : s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2269 : __atomic_acquire_fence();
2270 : return ret;
2271 : }
2272 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2273 : #endif
2274 :
2275 : #ifndef arch_atomic64_cmpxchg_release
2276 : static __always_inline s64
2277 : arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2278 : {
2279 : __atomic_release_fence();
2280 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
2281 : }
2282 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2283 : #endif
2284 :
2285 : #ifndef arch_atomic64_cmpxchg
2286 : static __always_inline s64
2287 : arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2288 : {
2289 : s64 ret;
2290 : __atomic_pre_full_fence();
2291 : ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2292 : __atomic_post_full_fence();
2293 : return ret;
2294 : }
2295 : #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2296 : #endif
2297 :
2298 : #endif /* arch_atomic64_cmpxchg_relaxed */
2299 :
2300 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2301 : #ifdef arch_atomic64_try_cmpxchg
2302 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2303 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2304 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2305 : #endif /* arch_atomic64_try_cmpxchg */
2306 :
2307 : #ifndef arch_atomic64_try_cmpxchg
2308 : static __always_inline bool
2309 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2310 : {
2311 : s64 r, o = *old;
2312 : r = arch_atomic64_cmpxchg(v, o, new);
2313 : if (unlikely(r != o))
2314 : *old = r;
2315 : return likely(r == o);
2316 : }
2317 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2318 : #endif
2319 :
2320 : #ifndef arch_atomic64_try_cmpxchg_acquire
2321 : static __always_inline bool
2322 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2323 : {
2324 : s64 r, o = *old;
2325 : r = arch_atomic64_cmpxchg_acquire(v, o, new);
2326 : if (unlikely(r != o))
2327 : *old = r;
2328 : return likely(r == o);
2329 : }
2330 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2331 : #endif
2332 :
2333 : #ifndef arch_atomic64_try_cmpxchg_release
2334 : static __always_inline bool
2335 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2336 : {
2337 : s64 r, o = *old;
2338 : r = arch_atomic64_cmpxchg_release(v, o, new);
2339 : if (unlikely(r != o))
2340 : *old = r;
2341 : return likely(r == o);
2342 : }
2343 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2344 : #endif
2345 :
2346 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2347 : static __always_inline bool
2348 : arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2349 : {
2350 : s64 r, o = *old;
2351 : r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2352 : if (unlikely(r != o))
2353 : *old = r;
2354 : return likely(r == o);
2355 : }
2356 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2357 : #endif
2358 :
2359 : #else /* arch_atomic64_try_cmpxchg_relaxed */
2360 :
2361 : #ifndef arch_atomic64_try_cmpxchg_acquire
2362 : static __always_inline bool
2363 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2364 : {
2365 : bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2366 : __atomic_acquire_fence();
2367 : return ret;
2368 : }
2369 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2370 : #endif
2371 :
2372 : #ifndef arch_atomic64_try_cmpxchg_release
2373 : static __always_inline bool
2374 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2375 : {
2376 : __atomic_release_fence();
2377 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2378 : }
2379 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2380 : #endif
2381 :
2382 : #ifndef arch_atomic64_try_cmpxchg
2383 : static __always_inline bool
2384 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2385 : {
2386 : bool ret;
2387 : __atomic_pre_full_fence();
2388 : ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2389 : __atomic_post_full_fence();
2390 : return ret;
2391 : }
2392 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2393 : #endif
2394 :
2395 : #endif /* arch_atomic64_try_cmpxchg_relaxed */
2396 :
2397 : #ifndef arch_atomic64_sub_and_test
2398 : /**
2399 : * arch_atomic64_sub_and_test - subtract value from variable and test result
2400 : * @i: integer value to subtract
2401 : * @v: pointer of type atomic64_t
2402 : *
2403 : * Atomically subtracts @i from @v and returns
2404 : * true if the result is zero, or false for all
2405 : * other cases.
2406 : */
2407 : static __always_inline bool
2408 : arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2409 : {
2410 : return arch_atomic64_sub_return(i, v) == 0;
2411 : }
2412 : #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2413 : #endif
2414 :
2415 : #ifndef arch_atomic64_dec_and_test
2416 : /**
2417 : * arch_atomic64_dec_and_test - decrement and test
2418 : * @v: pointer of type atomic64_t
2419 : *
2420 : * Atomically decrements @v by 1 and
2421 : * returns true if the result is 0, or false for all other
2422 : * cases.
2423 : */
2424 : static __always_inline bool
2425 : arch_atomic64_dec_and_test(atomic64_t *v)
2426 : {
2427 : return arch_atomic64_dec_return(v) == 0;
2428 : }
2429 : #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2430 : #endif
2431 :
2432 : #ifndef arch_atomic64_inc_and_test
2433 : /**
2434 : * arch_atomic64_inc_and_test - increment and test
2435 : * @v: pointer of type atomic64_t
2436 : *
2437 : * Atomically increments @v by 1
2438 : * and returns true if the result is zero, or false for all
2439 : * other cases.
2440 : */
2441 : static __always_inline bool
2442 : arch_atomic64_inc_and_test(atomic64_t *v)
2443 : {
2444 : return arch_atomic64_inc_return(v) == 0;
2445 : }
2446 : #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2447 : #endif
2448 :
2449 : #ifndef arch_atomic64_add_negative_relaxed
2450 : #ifdef arch_atomic64_add_negative
2451 : #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative
2452 : #define arch_atomic64_add_negative_release arch_atomic64_add_negative
2453 : #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative
2454 : #endif /* arch_atomic64_add_negative */
2455 :
2456 : #ifndef arch_atomic64_add_negative
2457 : /**
2458 : * arch_atomic64_add_negative - Add and test if negative
2459 : * @i: integer value to add
2460 : * @v: pointer of type atomic64_t
2461 : *
2462 : * Atomically adds @i to @v and returns true if the result is negative,
2463 : * or false when the result is greater than or equal to zero.
2464 : */
2465 : static __always_inline bool
2466 : arch_atomic64_add_negative(s64 i, atomic64_t *v)
2467 : {
2468 : return arch_atomic64_add_return(i, v) < 0;
2469 : }
2470 : #define arch_atomic64_add_negative arch_atomic64_add_negative
2471 : #endif
2472 :
2473 : #ifndef arch_atomic64_add_negative_acquire
2474 : /**
2475 : * arch_atomic64_add_negative_acquire - Add and test if negative
2476 : * @i: integer value to add
2477 : * @v: pointer of type atomic64_t
2478 : *
2479 : * Atomically adds @i to @v and returns true if the result is negative,
2480 : * or false when the result is greater than or equal to zero.
2481 : */
2482 : static __always_inline bool
2483 : arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
2484 : {
2485 : return arch_atomic64_add_return_acquire(i, v) < 0;
2486 : }
2487 : #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
2488 : #endif
2489 :
2490 : #ifndef arch_atomic64_add_negative_release
2491 : /**
2492 : * arch_atomic64_add_negative_release - Add and test if negative
2493 : * @i: integer value to add
2494 : * @v: pointer of type atomic64_t
2495 : *
2496 : * Atomically adds @i to @v and returns true if the result is negative,
2497 : * or false when the result is greater than or equal to zero.
2498 : */
2499 : static __always_inline bool
2500 : arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
2501 : {
2502 : return arch_atomic64_add_return_release(i, v) < 0;
2503 : }
2504 : #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
2505 : #endif
2506 :
2507 : #ifndef arch_atomic64_add_negative_relaxed
2508 : /**
2509 : * arch_atomic64_add_negative_relaxed - Add and test if negative
2510 : * @i: integer value to add
2511 : * @v: pointer of type atomic64_t
2512 : *
2513 : * Atomically adds @i to @v and returns true if the result is negative,
2514 : * or false when the result is greater than or equal to zero.
2515 : */
2516 : static __always_inline bool
2517 : arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
2518 : {
2519 : return arch_atomic64_add_return_relaxed(i, v) < 0;
2520 : }
2521 : #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed
2522 : #endif
2523 :
2524 : #else /* arch_atomic64_add_negative_relaxed */
2525 :
2526 : #ifndef arch_atomic64_add_negative_acquire
2527 : static __always_inline bool
2528 : arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
2529 : {
2530 : bool ret = arch_atomic64_add_negative_relaxed(i, v);
2531 : __atomic_acquire_fence();
2532 : return ret;
2533 : }
2534 : #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire
2535 : #endif
2536 :
2537 : #ifndef arch_atomic64_add_negative_release
2538 : static __always_inline bool
2539 : arch_atomic64_add_negative_release(s64 i, atomic64_t *v)
2540 : {
2541 : __atomic_release_fence();
2542 : return arch_atomic64_add_negative_relaxed(i, v);
2543 : }
2544 : #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release
2545 : #endif
2546 :
2547 : #ifndef arch_atomic64_add_negative
2548 : static __always_inline bool
2549 : arch_atomic64_add_negative(s64 i, atomic64_t *v)
2550 : {
2551 : bool ret;
2552 : __atomic_pre_full_fence();
2553 : ret = arch_atomic64_add_negative_relaxed(i, v);
2554 : __atomic_post_full_fence();
2555 : return ret;
2556 : }
2557 : #define arch_atomic64_add_negative arch_atomic64_add_negative
2558 : #endif
2559 :
2560 : #endif /* arch_atomic64_add_negative_relaxed */
2561 :
2562 : #ifndef arch_atomic64_fetch_add_unless
2563 : /**
2564 : * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2565 : * @v: pointer of type atomic64_t
2566 : * @a: the amount to add to v...
2567 : * @u: ...unless v is equal to u.
2568 : *
2569 : * Atomically adds @a to @v, so long as @v was not already @u.
2570 : * Returns original value of @v
2571 : */
2572 : static __always_inline s64
2573 : arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2574 : {
2575 0 : s64 c = arch_atomic64_read(v);
2576 :
2577 : do {
2578 0 : if (unlikely(c == u))
2579 : break;
2580 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2581 :
2582 : return c;
2583 : }
2584 : #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2585 : #endif
2586 :
2587 : #ifndef arch_atomic64_add_unless
2588 : /**
2589 : * arch_atomic64_add_unless - add unless the number is already a given value
2590 : * @v: pointer of type atomic64_t
2591 : * @a: the amount to add to v...
2592 : * @u: ...unless v is equal to u.
2593 : *
2594 : * Atomically adds @a to @v, if @v was not already @u.
2595 : * Returns true if the addition was done.
2596 : */
2597 : static __always_inline bool
2598 : arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2599 : {
2600 0 : return arch_atomic64_fetch_add_unless(v, a, u) != u;
2601 : }
2602 : #define arch_atomic64_add_unless arch_atomic64_add_unless
2603 : #endif
2604 :
2605 : #ifndef arch_atomic64_inc_not_zero
2606 : /**
2607 : * arch_atomic64_inc_not_zero - increment unless the number is zero
2608 : * @v: pointer of type atomic64_t
2609 : *
2610 : * Atomically increments @v by 1, if @v is non-zero.
2611 : * Returns true if the increment was done.
2612 : */
2613 : static __always_inline bool
2614 : arch_atomic64_inc_not_zero(atomic64_t *v)
2615 : {
2616 0 : return arch_atomic64_add_unless(v, 1, 0);
2617 : }
2618 : #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2619 : #endif
2620 :
2621 : #ifndef arch_atomic64_inc_unless_negative
2622 : static __always_inline bool
2623 : arch_atomic64_inc_unless_negative(atomic64_t *v)
2624 : {
2625 : s64 c = arch_atomic64_read(v);
2626 :
2627 : do {
2628 : if (unlikely(c < 0))
2629 : return false;
2630 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2631 :
2632 : return true;
2633 : }
2634 : #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2635 : #endif
2636 :
2637 : #ifndef arch_atomic64_dec_unless_positive
2638 : static __always_inline bool
2639 : arch_atomic64_dec_unless_positive(atomic64_t *v)
2640 : {
2641 : s64 c = arch_atomic64_read(v);
2642 :
2643 : do {
2644 : if (unlikely(c > 0))
2645 : return false;
2646 : } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2647 :
2648 : return true;
2649 : }
2650 : #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2651 : #endif
2652 :
2653 : #ifndef arch_atomic64_dec_if_positive
2654 : static __always_inline s64
2655 : arch_atomic64_dec_if_positive(atomic64_t *v)
2656 : {
2657 0 : s64 dec, c = arch_atomic64_read(v);
2658 :
2659 : do {
2660 0 : dec = c - 1;
2661 0 : if (unlikely(dec < 0))
2662 : break;
2663 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2664 :
2665 : return dec;
2666 : }
2667 : #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2668 : #endif
2669 :
2670 : #endif /* _LINUX_ATOMIC_FALLBACK_H */
2671 : // ad2e2b4d168dbc60a73922616047a9bfa446af36
|
