Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-fallback.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : #ifndef _LINUX_ATOMIC_FALLBACK_H
7 : #define _LINUX_ATOMIC_FALLBACK_H
8 :
9 : #include <linux/compiler.h>
10 :
11 : #ifndef arch_xchg_relaxed
12 : #define arch_xchg_acquire arch_xchg
13 : #define arch_xchg_release arch_xchg
14 : #define arch_xchg_relaxed arch_xchg
15 : #else /* arch_xchg_relaxed */
16 :
17 : #ifndef arch_xchg_acquire
18 : #define arch_xchg_acquire(...) \
19 : __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20 : #endif
21 :
22 : #ifndef arch_xchg_release
23 : #define arch_xchg_release(...) \
24 : __atomic_op_release(arch_xchg, __VA_ARGS__)
25 : #endif
26 :
27 : #ifndef arch_xchg
28 : #define arch_xchg(...) \
29 : __atomic_op_fence(arch_xchg, __VA_ARGS__)
30 : #endif
31 :
32 : #endif /* arch_xchg_relaxed */
33 :
34 : #ifndef arch_cmpxchg_relaxed
35 : #define arch_cmpxchg_acquire arch_cmpxchg
36 : #define arch_cmpxchg_release arch_cmpxchg
37 : #define arch_cmpxchg_relaxed arch_cmpxchg
38 : #else /* arch_cmpxchg_relaxed */
39 :
40 : #ifndef arch_cmpxchg_acquire
41 : #define arch_cmpxchg_acquire(...) \
42 : __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43 : #endif
44 :
45 : #ifndef arch_cmpxchg_release
46 : #define arch_cmpxchg_release(...) \
47 : __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48 : #endif
49 :
50 : #ifndef arch_cmpxchg
51 : #define arch_cmpxchg(...) \
52 : __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53 : #endif
54 :
55 : #endif /* arch_cmpxchg_relaxed */
56 :
57 : #ifndef arch_cmpxchg64_relaxed
58 : #define arch_cmpxchg64_acquire arch_cmpxchg64
59 : #define arch_cmpxchg64_release arch_cmpxchg64
60 : #define arch_cmpxchg64_relaxed arch_cmpxchg64
61 : #else /* arch_cmpxchg64_relaxed */
62 :
63 : #ifndef arch_cmpxchg64_acquire
64 : #define arch_cmpxchg64_acquire(...) \
65 : __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66 : #endif
67 :
68 : #ifndef arch_cmpxchg64_release
69 : #define arch_cmpxchg64_release(...) \
70 : __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71 : #endif
72 :
73 : #ifndef arch_cmpxchg64
74 : #define arch_cmpxchg64(...) \
75 : __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76 : #endif
77 :
78 : #endif /* arch_cmpxchg64_relaxed */
79 :
80 : #ifndef arch_try_cmpxchg_relaxed
81 : #ifdef arch_try_cmpxchg
82 : #define arch_try_cmpxchg_acquire arch_try_cmpxchg
83 : #define arch_try_cmpxchg_release arch_try_cmpxchg
84 : #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85 : #endif /* arch_try_cmpxchg */
86 :
87 : #ifndef arch_try_cmpxchg
88 : #define arch_try_cmpxchg(_ptr, _oldp, _new) \
89 : ({ \
90 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 : ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
92 : if (unlikely(___r != ___o)) \
93 : *___op = ___r; \
94 : likely(___r == ___o); \
95 : })
96 : #endif /* arch_try_cmpxchg */
97 :
98 : #ifndef arch_try_cmpxchg_acquire
99 : #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
100 : ({ \
101 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 : ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
103 : if (unlikely(___r != ___o)) \
104 : *___op = ___r; \
105 : likely(___r == ___o); \
106 : })
107 : #endif /* arch_try_cmpxchg_acquire */
108 :
109 : #ifndef arch_try_cmpxchg_release
110 : #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
111 : ({ \
112 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 : ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
114 : if (unlikely(___r != ___o)) \
115 : *___op = ___r; \
116 : likely(___r == ___o); \
117 : })
118 : #endif /* arch_try_cmpxchg_release */
119 :
120 : #ifndef arch_try_cmpxchg_relaxed
121 : #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
122 : ({ \
123 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 : ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
125 : if (unlikely(___r != ___o)) \
126 : *___op = ___r; \
127 : likely(___r == ___o); \
128 : })
129 : #endif /* arch_try_cmpxchg_relaxed */
130 :
131 : #else /* arch_try_cmpxchg_relaxed */
132 :
133 : #ifndef arch_try_cmpxchg_acquire
134 : #define arch_try_cmpxchg_acquire(...) \
135 : __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136 : #endif
137 :
138 : #ifndef arch_try_cmpxchg_release
139 : #define arch_try_cmpxchg_release(...) \
140 : __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
141 : #endif
142 :
143 : #ifndef arch_try_cmpxchg
144 : #define arch_try_cmpxchg(...) \
145 : __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
146 : #endif
147 :
148 : #endif /* arch_try_cmpxchg_relaxed */
149 :
150 : #ifndef arch_try_cmpxchg64_relaxed
151 : #ifdef arch_try_cmpxchg64
152 : #define arch_try_cmpxchg64_acquire arch_try_cmpxchg64
153 : #define arch_try_cmpxchg64_release arch_try_cmpxchg64
154 : #define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64
155 : #endif /* arch_try_cmpxchg64 */
156 :
157 : #ifndef arch_try_cmpxchg64
158 : #define arch_try_cmpxchg64(_ptr, _oldp, _new) \
159 : ({ \
160 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
161 : ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \
162 : if (unlikely(___r != ___o)) \
163 : *___op = ___r; \
164 : likely(___r == ___o); \
165 : })
166 : #endif /* arch_try_cmpxchg64 */
167 :
168 : #ifndef arch_try_cmpxchg64_acquire
169 : #define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
170 : ({ \
171 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
172 : ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \
173 : if (unlikely(___r != ___o)) \
174 : *___op = ___r; \
175 : likely(___r == ___o); \
176 : })
177 : #endif /* arch_try_cmpxchg64_acquire */
178 :
179 : #ifndef arch_try_cmpxchg64_release
180 : #define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \
181 : ({ \
182 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
183 : ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \
184 : if (unlikely(___r != ___o)) \
185 : *___op = ___r; \
186 : likely(___r == ___o); \
187 : })
188 : #endif /* arch_try_cmpxchg64_release */
189 :
190 : #ifndef arch_try_cmpxchg64_relaxed
191 : #define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
192 : ({ \
193 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
194 : ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
195 : if (unlikely(___r != ___o)) \
196 : *___op = ___r; \
197 : likely(___r == ___o); \
198 : })
199 : #endif /* arch_try_cmpxchg64_relaxed */
200 :
201 : #else /* arch_try_cmpxchg64_relaxed */
202 :
203 : #ifndef arch_try_cmpxchg64_acquire
204 : #define arch_try_cmpxchg64_acquire(...) \
205 : __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
206 : #endif
207 :
208 : #ifndef arch_try_cmpxchg64_release
209 : #define arch_try_cmpxchg64_release(...) \
210 : __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
211 : #endif
212 :
213 : #ifndef arch_try_cmpxchg64
214 : #define arch_try_cmpxchg64(...) \
215 : __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
216 : #endif
217 :
218 : #endif /* arch_try_cmpxchg64_relaxed */
219 :
220 : #ifndef arch_atomic_read_acquire
221 : static __always_inline int
222 : arch_atomic_read_acquire(const atomic_t *v)
223 : {
224 : int ret;
225 :
226 : if (__native_word(atomic_t)) {
227 0 : ret = smp_load_acquire(&(v)->counter);
228 : } else {
229 : ret = arch_atomic_read(v);
230 : __atomic_acquire_fence();
231 : }
232 :
233 : return ret;
234 : }
235 : #define arch_atomic_read_acquire arch_atomic_read_acquire
236 : #endif
237 :
238 : #ifndef arch_atomic_set_release
239 : static __always_inline void
240 : arch_atomic_set_release(atomic_t *v, int i)
241 : {
242 : if (__native_word(atomic_t)) {
243 0 : smp_store_release(&(v)->counter, i);
244 : } else {
245 : __atomic_release_fence();
246 : arch_atomic_set(v, i);
247 : }
248 : }
249 : #define arch_atomic_set_release arch_atomic_set_release
250 : #endif
251 :
252 : #ifndef arch_atomic_add_return_relaxed
253 : #define arch_atomic_add_return_acquire arch_atomic_add_return
254 : #define arch_atomic_add_return_release arch_atomic_add_return
255 : #define arch_atomic_add_return_relaxed arch_atomic_add_return
256 : #else /* arch_atomic_add_return_relaxed */
257 :
258 : #ifndef arch_atomic_add_return_acquire
259 : static __always_inline int
260 : arch_atomic_add_return_acquire(int i, atomic_t *v)
261 : {
262 : int ret = arch_atomic_add_return_relaxed(i, v);
263 : __atomic_acquire_fence();
264 : return ret;
265 : }
266 : #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
267 : #endif
268 :
269 : #ifndef arch_atomic_add_return_release
270 : static __always_inline int
271 : arch_atomic_add_return_release(int i, atomic_t *v)
272 : {
273 : __atomic_release_fence();
274 : return arch_atomic_add_return_relaxed(i, v);
275 : }
276 : #define arch_atomic_add_return_release arch_atomic_add_return_release
277 : #endif
278 :
279 : #ifndef arch_atomic_add_return
280 : static __always_inline int
281 : arch_atomic_add_return(int i, atomic_t *v)
282 : {
283 : int ret;
284 : __atomic_pre_full_fence();
285 : ret = arch_atomic_add_return_relaxed(i, v);
286 : __atomic_post_full_fence();
287 : return ret;
288 : }
289 : #define arch_atomic_add_return arch_atomic_add_return
290 : #endif
291 :
292 : #endif /* arch_atomic_add_return_relaxed */
293 :
294 : #ifndef arch_atomic_fetch_add_relaxed
295 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
296 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add
297 : #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
298 : #else /* arch_atomic_fetch_add_relaxed */
299 :
300 : #ifndef arch_atomic_fetch_add_acquire
301 : static __always_inline int
302 : arch_atomic_fetch_add_acquire(int i, atomic_t *v)
303 : {
304 : int ret = arch_atomic_fetch_add_relaxed(i, v);
305 : __atomic_acquire_fence();
306 : return ret;
307 : }
308 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
309 : #endif
310 :
311 : #ifndef arch_atomic_fetch_add_release
312 : static __always_inline int
313 : arch_atomic_fetch_add_release(int i, atomic_t *v)
314 : {
315 : __atomic_release_fence();
316 : return arch_atomic_fetch_add_relaxed(i, v);
317 : }
318 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
319 : #endif
320 :
321 : #ifndef arch_atomic_fetch_add
322 : static __always_inline int
323 : arch_atomic_fetch_add(int i, atomic_t *v)
324 : {
325 : int ret;
326 : __atomic_pre_full_fence();
327 : ret = arch_atomic_fetch_add_relaxed(i, v);
328 : __atomic_post_full_fence();
329 : return ret;
330 : }
331 : #define arch_atomic_fetch_add arch_atomic_fetch_add
332 : #endif
333 :
334 : #endif /* arch_atomic_fetch_add_relaxed */
335 :
336 : #ifndef arch_atomic_sub_return_relaxed
337 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return
338 : #define arch_atomic_sub_return_release arch_atomic_sub_return
339 : #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
340 : #else /* arch_atomic_sub_return_relaxed */
341 :
342 : #ifndef arch_atomic_sub_return_acquire
343 : static __always_inline int
344 : arch_atomic_sub_return_acquire(int i, atomic_t *v)
345 : {
346 : int ret = arch_atomic_sub_return_relaxed(i, v);
347 : __atomic_acquire_fence();
348 : return ret;
349 : }
350 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
351 : #endif
352 :
353 : #ifndef arch_atomic_sub_return_release
354 : static __always_inline int
355 : arch_atomic_sub_return_release(int i, atomic_t *v)
356 : {
357 : __atomic_release_fence();
358 : return arch_atomic_sub_return_relaxed(i, v);
359 : }
360 : #define arch_atomic_sub_return_release arch_atomic_sub_return_release
361 : #endif
362 :
363 : #ifndef arch_atomic_sub_return
364 : static __always_inline int
365 : arch_atomic_sub_return(int i, atomic_t *v)
366 : {
367 : int ret;
368 : __atomic_pre_full_fence();
369 : ret = arch_atomic_sub_return_relaxed(i, v);
370 : __atomic_post_full_fence();
371 : return ret;
372 : }
373 : #define arch_atomic_sub_return arch_atomic_sub_return
374 : #endif
375 :
376 : #endif /* arch_atomic_sub_return_relaxed */
377 :
378 : #ifndef arch_atomic_fetch_sub_relaxed
379 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
380 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
381 : #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
382 : #else /* arch_atomic_fetch_sub_relaxed */
383 :
384 : #ifndef arch_atomic_fetch_sub_acquire
385 : static __always_inline int
386 : arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
387 : {
388 : int ret = arch_atomic_fetch_sub_relaxed(i, v);
389 : __atomic_acquire_fence();
390 : return ret;
391 : }
392 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
393 : #endif
394 :
395 : #ifndef arch_atomic_fetch_sub_release
396 : static __always_inline int
397 : arch_atomic_fetch_sub_release(int i, atomic_t *v)
398 : {
399 : __atomic_release_fence();
400 : return arch_atomic_fetch_sub_relaxed(i, v);
401 : }
402 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
403 : #endif
404 :
405 : #ifndef arch_atomic_fetch_sub
406 : static __always_inline int
407 : arch_atomic_fetch_sub(int i, atomic_t *v)
408 : {
409 : int ret;
410 : __atomic_pre_full_fence();
411 : ret = arch_atomic_fetch_sub_relaxed(i, v);
412 : __atomic_post_full_fence();
413 : return ret;
414 : }
415 : #define arch_atomic_fetch_sub arch_atomic_fetch_sub
416 : #endif
417 :
418 : #endif /* arch_atomic_fetch_sub_relaxed */
419 :
420 : #ifndef arch_atomic_inc
421 : static __always_inline void
422 : arch_atomic_inc(atomic_t *v)
423 : {
424 : arch_atomic_add(1, v);
425 : }
426 : #define arch_atomic_inc arch_atomic_inc
427 : #endif
428 :
429 : #ifndef arch_atomic_inc_return_relaxed
430 : #ifdef arch_atomic_inc_return
431 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return
432 : #define arch_atomic_inc_return_release arch_atomic_inc_return
433 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
434 : #endif /* arch_atomic_inc_return */
435 :
436 : #ifndef arch_atomic_inc_return
437 : static __always_inline int
438 : arch_atomic_inc_return(atomic_t *v)
439 : {
440 5 : return arch_atomic_add_return(1, v);
441 : }
442 : #define arch_atomic_inc_return arch_atomic_inc_return
443 : #endif
444 :
445 : #ifndef arch_atomic_inc_return_acquire
446 : static __always_inline int
447 : arch_atomic_inc_return_acquire(atomic_t *v)
448 : {
449 : return arch_atomic_add_return_acquire(1, v);
450 : }
451 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
452 : #endif
453 :
454 : #ifndef arch_atomic_inc_return_release
455 : static __always_inline int
456 : arch_atomic_inc_return_release(atomic_t *v)
457 : {
458 : return arch_atomic_add_return_release(1, v);
459 : }
460 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
461 : #endif
462 :
463 : #ifndef arch_atomic_inc_return_relaxed
464 : static __always_inline int
465 : arch_atomic_inc_return_relaxed(atomic_t *v)
466 : {
467 0 : return arch_atomic_add_return_relaxed(1, v);
468 : }
469 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
470 : #endif
471 :
472 : #else /* arch_atomic_inc_return_relaxed */
473 :
474 : #ifndef arch_atomic_inc_return_acquire
475 : static __always_inline int
476 : arch_atomic_inc_return_acquire(atomic_t *v)
477 : {
478 : int ret = arch_atomic_inc_return_relaxed(v);
479 : __atomic_acquire_fence();
480 : return ret;
481 : }
482 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
483 : #endif
484 :
485 : #ifndef arch_atomic_inc_return_release
486 : static __always_inline int
487 : arch_atomic_inc_return_release(atomic_t *v)
488 : {
489 : __atomic_release_fence();
490 : return arch_atomic_inc_return_relaxed(v);
491 : }
492 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
493 : #endif
494 :
495 : #ifndef arch_atomic_inc_return
496 : static __always_inline int
497 : arch_atomic_inc_return(atomic_t *v)
498 : {
499 : int ret;
500 : __atomic_pre_full_fence();
501 : ret = arch_atomic_inc_return_relaxed(v);
502 : __atomic_post_full_fence();
503 : return ret;
504 : }
505 : #define arch_atomic_inc_return arch_atomic_inc_return
506 : #endif
507 :
508 : #endif /* arch_atomic_inc_return_relaxed */
509 :
510 : #ifndef arch_atomic_fetch_inc_relaxed
511 : #ifdef arch_atomic_fetch_inc
512 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
513 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
514 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
515 : #endif /* arch_atomic_fetch_inc */
516 :
517 : #ifndef arch_atomic_fetch_inc
518 : static __always_inline int
519 : arch_atomic_fetch_inc(atomic_t *v)
520 : {
521 0 : return arch_atomic_fetch_add(1, v);
522 : }
523 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
524 : #endif
525 :
526 : #ifndef arch_atomic_fetch_inc_acquire
527 : static __always_inline int
528 : arch_atomic_fetch_inc_acquire(atomic_t *v)
529 : {
530 : return arch_atomic_fetch_add_acquire(1, v);
531 : }
532 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
533 : #endif
534 :
535 : #ifndef arch_atomic_fetch_inc_release
536 : static __always_inline int
537 : arch_atomic_fetch_inc_release(atomic_t *v)
538 : {
539 : return arch_atomic_fetch_add_release(1, v);
540 : }
541 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
542 : #endif
543 :
544 : #ifndef arch_atomic_fetch_inc_relaxed
545 : static __always_inline int
546 : arch_atomic_fetch_inc_relaxed(atomic_t *v)
547 : {
548 : return arch_atomic_fetch_add_relaxed(1, v);
549 : }
550 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
551 : #endif
552 :
553 : #else /* arch_atomic_fetch_inc_relaxed */
554 :
555 : #ifndef arch_atomic_fetch_inc_acquire
556 : static __always_inline int
557 : arch_atomic_fetch_inc_acquire(atomic_t *v)
558 : {
559 : int ret = arch_atomic_fetch_inc_relaxed(v);
560 : __atomic_acquire_fence();
561 : return ret;
562 : }
563 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
564 : #endif
565 :
566 : #ifndef arch_atomic_fetch_inc_release
567 : static __always_inline int
568 : arch_atomic_fetch_inc_release(atomic_t *v)
569 : {
570 : __atomic_release_fence();
571 : return arch_atomic_fetch_inc_relaxed(v);
572 : }
573 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
574 : #endif
575 :
576 : #ifndef arch_atomic_fetch_inc
577 : static __always_inline int
578 : arch_atomic_fetch_inc(atomic_t *v)
579 : {
580 : int ret;
581 : __atomic_pre_full_fence();
582 : ret = arch_atomic_fetch_inc_relaxed(v);
583 : __atomic_post_full_fence();
584 : return ret;
585 : }
586 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
587 : #endif
588 :
589 : #endif /* arch_atomic_fetch_inc_relaxed */
590 :
591 : #ifndef arch_atomic_dec
592 : static __always_inline void
593 : arch_atomic_dec(atomic_t *v)
594 : {
595 : arch_atomic_sub(1, v);
596 : }
597 : #define arch_atomic_dec arch_atomic_dec
598 : #endif
599 :
600 : #ifndef arch_atomic_dec_return_relaxed
601 : #ifdef arch_atomic_dec_return
602 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return
603 : #define arch_atomic_dec_return_release arch_atomic_dec_return
604 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
605 : #endif /* arch_atomic_dec_return */
606 :
607 : #ifndef arch_atomic_dec_return
608 : static __always_inline int
609 : arch_atomic_dec_return(atomic_t *v)
610 : {
611 0 : return arch_atomic_sub_return(1, v);
612 : }
613 : #define arch_atomic_dec_return arch_atomic_dec_return
614 : #endif
615 :
616 : #ifndef arch_atomic_dec_return_acquire
617 : static __always_inline int
618 : arch_atomic_dec_return_acquire(atomic_t *v)
619 : {
620 : return arch_atomic_sub_return_acquire(1, v);
621 : }
622 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
623 : #endif
624 :
625 : #ifndef arch_atomic_dec_return_release
626 : static __always_inline int
627 : arch_atomic_dec_return_release(atomic_t *v)
628 : {
629 : return arch_atomic_sub_return_release(1, v);
630 : }
631 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
632 : #endif
633 :
634 : #ifndef arch_atomic_dec_return_relaxed
635 : static __always_inline int
636 : arch_atomic_dec_return_relaxed(atomic_t *v)
637 : {
638 0 : return arch_atomic_sub_return_relaxed(1, v);
639 : }
640 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
641 : #endif
642 :
643 : #else /* arch_atomic_dec_return_relaxed */
644 :
645 : #ifndef arch_atomic_dec_return_acquire
646 : static __always_inline int
647 : arch_atomic_dec_return_acquire(atomic_t *v)
648 : {
649 : int ret = arch_atomic_dec_return_relaxed(v);
650 : __atomic_acquire_fence();
651 : return ret;
652 : }
653 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
654 : #endif
655 :
656 : #ifndef arch_atomic_dec_return_release
657 : static __always_inline int
658 : arch_atomic_dec_return_release(atomic_t *v)
659 : {
660 : __atomic_release_fence();
661 : return arch_atomic_dec_return_relaxed(v);
662 : }
663 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
664 : #endif
665 :
666 : #ifndef arch_atomic_dec_return
667 : static __always_inline int
668 : arch_atomic_dec_return(atomic_t *v)
669 : {
670 : int ret;
671 : __atomic_pre_full_fence();
672 : ret = arch_atomic_dec_return_relaxed(v);
673 : __atomic_post_full_fence();
674 : return ret;
675 : }
676 : #define arch_atomic_dec_return arch_atomic_dec_return
677 : #endif
678 :
679 : #endif /* arch_atomic_dec_return_relaxed */
680 :
681 : #ifndef arch_atomic_fetch_dec_relaxed
682 : #ifdef arch_atomic_fetch_dec
683 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
684 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
685 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
686 : #endif /* arch_atomic_fetch_dec */
687 :
688 : #ifndef arch_atomic_fetch_dec
689 : static __always_inline int
690 : arch_atomic_fetch_dec(atomic_t *v)
691 : {
692 : return arch_atomic_fetch_sub(1, v);
693 : }
694 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
695 : #endif
696 :
697 : #ifndef arch_atomic_fetch_dec_acquire
698 : static __always_inline int
699 : arch_atomic_fetch_dec_acquire(atomic_t *v)
700 : {
701 : return arch_atomic_fetch_sub_acquire(1, v);
702 : }
703 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
704 : #endif
705 :
706 : #ifndef arch_atomic_fetch_dec_release
707 : static __always_inline int
708 : arch_atomic_fetch_dec_release(atomic_t *v)
709 : {
710 : return arch_atomic_fetch_sub_release(1, v);
711 : }
712 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
713 : #endif
714 :
715 : #ifndef arch_atomic_fetch_dec_relaxed
716 : static __always_inline int
717 : arch_atomic_fetch_dec_relaxed(atomic_t *v)
718 : {
719 : return arch_atomic_fetch_sub_relaxed(1, v);
720 : }
721 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
722 : #endif
723 :
724 : #else /* arch_atomic_fetch_dec_relaxed */
725 :
726 : #ifndef arch_atomic_fetch_dec_acquire
727 : static __always_inline int
728 : arch_atomic_fetch_dec_acquire(atomic_t *v)
729 : {
730 : int ret = arch_atomic_fetch_dec_relaxed(v);
731 : __atomic_acquire_fence();
732 : return ret;
733 : }
734 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
735 : #endif
736 :
737 : #ifndef arch_atomic_fetch_dec_release
738 : static __always_inline int
739 : arch_atomic_fetch_dec_release(atomic_t *v)
740 : {
741 : __atomic_release_fence();
742 : return arch_atomic_fetch_dec_relaxed(v);
743 : }
744 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
745 : #endif
746 :
747 : #ifndef arch_atomic_fetch_dec
748 : static __always_inline int
749 : arch_atomic_fetch_dec(atomic_t *v)
750 : {
751 : int ret;
752 : __atomic_pre_full_fence();
753 : ret = arch_atomic_fetch_dec_relaxed(v);
754 : __atomic_post_full_fence();
755 : return ret;
756 : }
757 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
758 : #endif
759 :
760 : #endif /* arch_atomic_fetch_dec_relaxed */
761 :
762 : #ifndef arch_atomic_fetch_and_relaxed
763 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
764 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and
765 : #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
766 : #else /* arch_atomic_fetch_and_relaxed */
767 :
768 : #ifndef arch_atomic_fetch_and_acquire
769 : static __always_inline int
770 : arch_atomic_fetch_and_acquire(int i, atomic_t *v)
771 : {
772 : int ret = arch_atomic_fetch_and_relaxed(i, v);
773 : __atomic_acquire_fence();
774 : return ret;
775 : }
776 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
777 : #endif
778 :
779 : #ifndef arch_atomic_fetch_and_release
780 : static __always_inline int
781 : arch_atomic_fetch_and_release(int i, atomic_t *v)
782 : {
783 : __atomic_release_fence();
784 : return arch_atomic_fetch_and_relaxed(i, v);
785 : }
786 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
787 : #endif
788 :
789 : #ifndef arch_atomic_fetch_and
790 : static __always_inline int
791 : arch_atomic_fetch_and(int i, atomic_t *v)
792 : {
793 : int ret;
794 : __atomic_pre_full_fence();
795 : ret = arch_atomic_fetch_and_relaxed(i, v);
796 : __atomic_post_full_fence();
797 : return ret;
798 : }
799 : #define arch_atomic_fetch_and arch_atomic_fetch_and
800 : #endif
801 :
802 : #endif /* arch_atomic_fetch_and_relaxed */
803 :
804 : #ifndef arch_atomic_andnot
805 : static __always_inline void
806 : arch_atomic_andnot(int i, atomic_t *v)
807 : {
808 0 : arch_atomic_and(~i, v);
809 : }
810 : #define arch_atomic_andnot arch_atomic_andnot
811 : #endif
812 :
813 : #ifndef arch_atomic_fetch_andnot_relaxed
814 : #ifdef arch_atomic_fetch_andnot
815 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
816 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
817 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
818 : #endif /* arch_atomic_fetch_andnot */
819 :
820 : #ifndef arch_atomic_fetch_andnot
821 : static __always_inline int
822 : arch_atomic_fetch_andnot(int i, atomic_t *v)
823 : {
824 : return arch_atomic_fetch_and(~i, v);
825 : }
826 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
827 : #endif
828 :
829 : #ifndef arch_atomic_fetch_andnot_acquire
830 : static __always_inline int
831 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
832 : {
833 : return arch_atomic_fetch_and_acquire(~i, v);
834 : }
835 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
836 : #endif
837 :
838 : #ifndef arch_atomic_fetch_andnot_release
839 : static __always_inline int
840 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
841 : {
842 : return arch_atomic_fetch_and_release(~i, v);
843 : }
844 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
845 : #endif
846 :
847 : #ifndef arch_atomic_fetch_andnot_relaxed
848 : static __always_inline int
849 : arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
850 : {
851 : return arch_atomic_fetch_and_relaxed(~i, v);
852 : }
853 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
854 : #endif
855 :
856 : #else /* arch_atomic_fetch_andnot_relaxed */
857 :
858 : #ifndef arch_atomic_fetch_andnot_acquire
859 : static __always_inline int
860 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
861 : {
862 : int ret = arch_atomic_fetch_andnot_relaxed(i, v);
863 : __atomic_acquire_fence();
864 : return ret;
865 : }
866 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
867 : #endif
868 :
869 : #ifndef arch_atomic_fetch_andnot_release
870 : static __always_inline int
871 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
872 : {
873 : __atomic_release_fence();
874 : return arch_atomic_fetch_andnot_relaxed(i, v);
875 : }
876 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
877 : #endif
878 :
879 : #ifndef arch_atomic_fetch_andnot
880 : static __always_inline int
881 : arch_atomic_fetch_andnot(int i, atomic_t *v)
882 : {
883 : int ret;
884 : __atomic_pre_full_fence();
885 : ret = arch_atomic_fetch_andnot_relaxed(i, v);
886 : __atomic_post_full_fence();
887 : return ret;
888 : }
889 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
890 : #endif
891 :
892 : #endif /* arch_atomic_fetch_andnot_relaxed */
893 :
894 : #ifndef arch_atomic_fetch_or_relaxed
895 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
896 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or
897 : #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
898 : #else /* arch_atomic_fetch_or_relaxed */
899 :
900 : #ifndef arch_atomic_fetch_or_acquire
901 : static __always_inline int
902 : arch_atomic_fetch_or_acquire(int i, atomic_t *v)
903 : {
904 : int ret = arch_atomic_fetch_or_relaxed(i, v);
905 : __atomic_acquire_fence();
906 : return ret;
907 : }
908 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
909 : #endif
910 :
911 : #ifndef arch_atomic_fetch_or_release
912 : static __always_inline int
913 : arch_atomic_fetch_or_release(int i, atomic_t *v)
914 : {
915 : __atomic_release_fence();
916 : return arch_atomic_fetch_or_relaxed(i, v);
917 : }
918 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
919 : #endif
920 :
921 : #ifndef arch_atomic_fetch_or
922 : static __always_inline int
923 : arch_atomic_fetch_or(int i, atomic_t *v)
924 : {
925 : int ret;
926 : __atomic_pre_full_fence();
927 : ret = arch_atomic_fetch_or_relaxed(i, v);
928 : __atomic_post_full_fence();
929 : return ret;
930 : }
931 : #define arch_atomic_fetch_or arch_atomic_fetch_or
932 : #endif
933 :
934 : #endif /* arch_atomic_fetch_or_relaxed */
935 :
936 : #ifndef arch_atomic_fetch_xor_relaxed
937 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
938 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
939 : #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
940 : #else /* arch_atomic_fetch_xor_relaxed */
941 :
942 : #ifndef arch_atomic_fetch_xor_acquire
943 : static __always_inline int
944 : arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
945 : {
946 : int ret = arch_atomic_fetch_xor_relaxed(i, v);
947 : __atomic_acquire_fence();
948 : return ret;
949 : }
950 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
951 : #endif
952 :
953 : #ifndef arch_atomic_fetch_xor_release
954 : static __always_inline int
955 : arch_atomic_fetch_xor_release(int i, atomic_t *v)
956 : {
957 : __atomic_release_fence();
958 : return arch_atomic_fetch_xor_relaxed(i, v);
959 : }
960 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
961 : #endif
962 :
963 : #ifndef arch_atomic_fetch_xor
964 : static __always_inline int
965 : arch_atomic_fetch_xor(int i, atomic_t *v)
966 : {
967 : int ret;
968 : __atomic_pre_full_fence();
969 : ret = arch_atomic_fetch_xor_relaxed(i, v);
970 : __atomic_post_full_fence();
971 : return ret;
972 : }
973 : #define arch_atomic_fetch_xor arch_atomic_fetch_xor
974 : #endif
975 :
976 : #endif /* arch_atomic_fetch_xor_relaxed */
977 :
978 : #ifndef arch_atomic_xchg_relaxed
979 : #define arch_atomic_xchg_acquire arch_atomic_xchg
980 : #define arch_atomic_xchg_release arch_atomic_xchg
981 : #define arch_atomic_xchg_relaxed arch_atomic_xchg
982 : #else /* arch_atomic_xchg_relaxed */
983 :
984 : #ifndef arch_atomic_xchg_acquire
985 : static __always_inline int
986 : arch_atomic_xchg_acquire(atomic_t *v, int i)
987 : {
988 : int ret = arch_atomic_xchg_relaxed(v, i);
989 : __atomic_acquire_fence();
990 : return ret;
991 : }
992 : #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
993 : #endif
994 :
995 : #ifndef arch_atomic_xchg_release
996 : static __always_inline int
997 : arch_atomic_xchg_release(atomic_t *v, int i)
998 : {
999 : __atomic_release_fence();
1000 : return arch_atomic_xchg_relaxed(v, i);
1001 : }
1002 : #define arch_atomic_xchg_release arch_atomic_xchg_release
1003 : #endif
1004 :
1005 : #ifndef arch_atomic_xchg
1006 : static __always_inline int
1007 : arch_atomic_xchg(atomic_t *v, int i)
1008 : {
1009 : int ret;
1010 : __atomic_pre_full_fence();
1011 : ret = arch_atomic_xchg_relaxed(v, i);
1012 : __atomic_post_full_fence();
1013 : return ret;
1014 : }
1015 : #define arch_atomic_xchg arch_atomic_xchg
1016 : #endif
1017 :
1018 : #endif /* arch_atomic_xchg_relaxed */
1019 :
1020 : #ifndef arch_atomic_cmpxchg_relaxed
1021 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
1022 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
1023 : #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
1024 : #else /* arch_atomic_cmpxchg_relaxed */
1025 :
1026 : #ifndef arch_atomic_cmpxchg_acquire
1027 : static __always_inline int
1028 : arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
1029 : {
1030 : int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1031 : __atomic_acquire_fence();
1032 : return ret;
1033 : }
1034 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
1035 : #endif
1036 :
1037 : #ifndef arch_atomic_cmpxchg_release
1038 : static __always_inline int
1039 : arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
1040 : {
1041 : __atomic_release_fence();
1042 : return arch_atomic_cmpxchg_relaxed(v, old, new);
1043 : }
1044 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
1045 : #endif
1046 :
1047 : #ifndef arch_atomic_cmpxchg
1048 : static __always_inline int
1049 : arch_atomic_cmpxchg(atomic_t *v, int old, int new)
1050 : {
1051 : int ret;
1052 : __atomic_pre_full_fence();
1053 : ret = arch_atomic_cmpxchg_relaxed(v, old, new);
1054 : __atomic_post_full_fence();
1055 : return ret;
1056 : }
1057 : #define arch_atomic_cmpxchg arch_atomic_cmpxchg
1058 : #endif
1059 :
1060 : #endif /* arch_atomic_cmpxchg_relaxed */
1061 :
1062 : #ifndef arch_atomic_try_cmpxchg_relaxed
1063 : #ifdef arch_atomic_try_cmpxchg
1064 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
1065 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
1066 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
1067 : #endif /* arch_atomic_try_cmpxchg */
1068 :
1069 : #ifndef arch_atomic_try_cmpxchg
1070 : static __always_inline bool
1071 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1072 : {
1073 : int r, o = *old;
1074 : r = arch_atomic_cmpxchg(v, o, new);
1075 : if (unlikely(r != o))
1076 : *old = r;
1077 : return likely(r == o);
1078 : }
1079 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1080 : #endif
1081 :
1082 : #ifndef arch_atomic_try_cmpxchg_acquire
1083 : static __always_inline bool
1084 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1085 : {
1086 : int r, o = *old;
1087 : r = arch_atomic_cmpxchg_acquire(v, o, new);
1088 : if (unlikely(r != o))
1089 : *old = r;
1090 : return likely(r == o);
1091 : }
1092 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1093 : #endif
1094 :
1095 : #ifndef arch_atomic_try_cmpxchg_release
1096 : static __always_inline bool
1097 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1098 : {
1099 : int r, o = *old;
1100 : r = arch_atomic_cmpxchg_release(v, o, new);
1101 : if (unlikely(r != o))
1102 : *old = r;
1103 : return likely(r == o);
1104 : }
1105 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1106 : #endif
1107 :
1108 : #ifndef arch_atomic_try_cmpxchg_relaxed
1109 : static __always_inline bool
1110 : arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1111 : {
1112 : int r, o = *old;
1113 : r = arch_atomic_cmpxchg_relaxed(v, o, new);
1114 : if (unlikely(r != o))
1115 : *old = r;
1116 : return likely(r == o);
1117 : }
1118 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1119 : #endif
1120 :
1121 : #else /* arch_atomic_try_cmpxchg_relaxed */
1122 :
1123 : #ifndef arch_atomic_try_cmpxchg_acquire
1124 : static __always_inline bool
1125 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1126 : {
1127 : bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1128 : __atomic_acquire_fence();
1129 : return ret;
1130 : }
1131 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1132 : #endif
1133 :
1134 : #ifndef arch_atomic_try_cmpxchg_release
1135 : static __always_inline bool
1136 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1137 : {
1138 : __atomic_release_fence();
1139 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1140 : }
1141 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1142 : #endif
1143 :
1144 : #ifndef arch_atomic_try_cmpxchg
1145 : static __always_inline bool
1146 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1147 : {
1148 : bool ret;
1149 : __atomic_pre_full_fence();
1150 : ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1151 : __atomic_post_full_fence();
1152 : return ret;
1153 : }
1154 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1155 : #endif
1156 :
1157 : #endif /* arch_atomic_try_cmpxchg_relaxed */
1158 :
1159 : #ifndef arch_atomic_sub_and_test
1160 : /**
1161 : * arch_atomic_sub_and_test - subtract value from variable and test result
1162 : * @i: integer value to subtract
1163 : * @v: pointer of type atomic_t
1164 : *
1165 : * Atomically subtracts @i from @v and returns
1166 : * true if the result is zero, or false for all
1167 : * other cases.
1168 : */
1169 : static __always_inline bool
1170 : arch_atomic_sub_and_test(int i, atomic_t *v)
1171 : {
1172 : return arch_atomic_sub_return(i, v) == 0;
1173 : }
1174 : #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1175 : #endif
1176 :
1177 : #ifndef arch_atomic_dec_and_test
1178 : /**
1179 : * arch_atomic_dec_and_test - decrement and test
1180 : * @v: pointer of type atomic_t
1181 : *
1182 : * Atomically decrements @v by 1 and
1183 : * returns true if the result is 0, or false for all other
1184 : * cases.
1185 : */
1186 : static __always_inline bool
1187 : arch_atomic_dec_and_test(atomic_t *v)
1188 : {
1189 : return arch_atomic_dec_return(v) == 0;
1190 : }
1191 : #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1192 : #endif
1193 :
1194 : #ifndef arch_atomic_inc_and_test
1195 : /**
1196 : * arch_atomic_inc_and_test - increment and test
1197 : * @v: pointer of type atomic_t
1198 : *
1199 : * Atomically increments @v by 1
1200 : * and returns true if the result is zero, or false for all
1201 : * other cases.
1202 : */
1203 : static __always_inline bool
1204 : arch_atomic_inc_and_test(atomic_t *v)
1205 : {
1206 : return arch_atomic_inc_return(v) == 0;
1207 : }
1208 : #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1209 : #endif
1210 :
1211 : #ifndef arch_atomic_add_negative
1212 : /**
1213 : * arch_atomic_add_negative - add and test if negative
1214 : * @i: integer value to add
1215 : * @v: pointer of type atomic_t
1216 : *
1217 : * Atomically adds @i to @v and returns true
1218 : * if the result is negative, or false when
1219 : * result is greater than or equal to zero.
1220 : */
1221 : static __always_inline bool
1222 : arch_atomic_add_negative(int i, atomic_t *v)
1223 : {
1224 : return arch_atomic_add_return(i, v) < 0;
1225 : }
1226 : #define arch_atomic_add_negative arch_atomic_add_negative
1227 : #endif
1228 :
1229 : #ifndef arch_atomic_fetch_add_unless
1230 : /**
1231 : * arch_atomic_fetch_add_unless - add unless the number is already a given value
1232 : * @v: pointer of type atomic_t
1233 : * @a: the amount to add to v...
1234 : * @u: ...unless v is equal to u.
1235 : *
1236 : * Atomically adds @a to @v, so long as @v was not already @u.
1237 : * Returns original value of @v
1238 : */
1239 : static __always_inline int
1240 : arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1241 : {
1242 955 : int c = arch_atomic_read(v);
1243 :
1244 : do {
1245 955 : if (unlikely(c == u))
1246 : break;
1247 1808 : } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1248 :
1249 : return c;
1250 : }
1251 : #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1252 : #endif
1253 :
1254 : #ifndef arch_atomic_add_unless
1255 : /**
1256 : * arch_atomic_add_unless - add unless the number is already a given value
1257 : * @v: pointer of type atomic_t
1258 : * @a: the amount to add to v...
1259 : * @u: ...unless v is equal to u.
1260 : *
1261 : * Atomically adds @a to @v, if @v was not already @u.
1262 : * Returns true if the addition was done.
1263 : */
1264 : static __always_inline bool
1265 : arch_atomic_add_unless(atomic_t *v, int a, int u)
1266 : {
1267 955 : return arch_atomic_fetch_add_unless(v, a, u) != u;
1268 : }
1269 : #define arch_atomic_add_unless arch_atomic_add_unless
1270 : #endif
1271 :
1272 : #ifndef arch_atomic_inc_not_zero
1273 : /**
1274 : * arch_atomic_inc_not_zero - increment unless the number is zero
1275 : * @v: pointer of type atomic_t
1276 : *
1277 : * Atomically increments @v by 1, if @v is non-zero.
1278 : * Returns true if the increment was done.
1279 : */
1280 : static __always_inline bool
1281 : arch_atomic_inc_not_zero(atomic_t *v)
1282 : {
1283 0 : return arch_atomic_add_unless(v, 1, 0);
1284 : }
1285 : #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1286 : #endif
1287 :
1288 : #ifndef arch_atomic_inc_unless_negative
1289 : static __always_inline bool
1290 : arch_atomic_inc_unless_negative(atomic_t *v)
1291 : {
1292 0 : int c = arch_atomic_read(v);
1293 :
1294 : do {
1295 0 : if (unlikely(c < 0))
1296 : return false;
1297 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1298 :
1299 : return true;
1300 : }
1301 : #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1302 : #endif
1303 :
1304 : #ifndef arch_atomic_dec_unless_positive
1305 : static __always_inline bool
1306 : arch_atomic_dec_unless_positive(atomic_t *v)
1307 : {
1308 0 : int c = arch_atomic_read(v);
1309 :
1310 : do {
1311 0 : if (unlikely(c > 0))
1312 : return false;
1313 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1314 :
1315 : return true;
1316 : }
1317 : #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1318 : #endif
1319 :
1320 : #ifndef arch_atomic_dec_if_positive
1321 : static __always_inline int
1322 : arch_atomic_dec_if_positive(atomic_t *v)
1323 : {
1324 0 : int dec, c = arch_atomic_read(v);
1325 :
1326 : do {
1327 0 : dec = c - 1;
1328 0 : if (unlikely(dec < 0))
1329 : break;
1330 0 : } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1331 :
1332 : return dec;
1333 : }
1334 : #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1335 : #endif
1336 :
1337 : #ifdef CONFIG_GENERIC_ATOMIC64
1338 : #include <asm-generic/atomic64.h>
1339 : #endif
1340 :
1341 : #ifndef arch_atomic64_read_acquire
1342 : static __always_inline s64
1343 : arch_atomic64_read_acquire(const atomic64_t *v)
1344 : {
1345 : s64 ret;
1346 :
1347 : if (__native_word(atomic64_t)) {
1348 : ret = smp_load_acquire(&(v)->counter);
1349 : } else {
1350 : ret = arch_atomic64_read(v);
1351 : __atomic_acquire_fence();
1352 : }
1353 :
1354 : return ret;
1355 : }
1356 : #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1357 : #endif
1358 :
1359 : #ifndef arch_atomic64_set_release
1360 : static __always_inline void
1361 : arch_atomic64_set_release(atomic64_t *v, s64 i)
1362 : {
1363 : if (__native_word(atomic64_t)) {
1364 : smp_store_release(&(v)->counter, i);
1365 : } else {
1366 : __atomic_release_fence();
1367 : arch_atomic64_set(v, i);
1368 : }
1369 : }
1370 : #define arch_atomic64_set_release arch_atomic64_set_release
1371 : #endif
1372 :
1373 : #ifndef arch_atomic64_add_return_relaxed
1374 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1375 : #define arch_atomic64_add_return_release arch_atomic64_add_return
1376 : #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1377 : #else /* arch_atomic64_add_return_relaxed */
1378 :
1379 : #ifndef arch_atomic64_add_return_acquire
1380 : static __always_inline s64
1381 : arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1382 : {
1383 : s64 ret = arch_atomic64_add_return_relaxed(i, v);
1384 : __atomic_acquire_fence();
1385 : return ret;
1386 : }
1387 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1388 : #endif
1389 :
1390 : #ifndef arch_atomic64_add_return_release
1391 : static __always_inline s64
1392 : arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1393 : {
1394 : __atomic_release_fence();
1395 : return arch_atomic64_add_return_relaxed(i, v);
1396 : }
1397 : #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1398 : #endif
1399 :
1400 : #ifndef arch_atomic64_add_return
1401 : static __always_inline s64
1402 : arch_atomic64_add_return(s64 i, atomic64_t *v)
1403 : {
1404 : s64 ret;
1405 : __atomic_pre_full_fence();
1406 : ret = arch_atomic64_add_return_relaxed(i, v);
1407 : __atomic_post_full_fence();
1408 : return ret;
1409 : }
1410 : #define arch_atomic64_add_return arch_atomic64_add_return
1411 : #endif
1412 :
1413 : #endif /* arch_atomic64_add_return_relaxed */
1414 :
1415 : #ifndef arch_atomic64_fetch_add_relaxed
1416 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1417 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1418 : #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1419 : #else /* arch_atomic64_fetch_add_relaxed */
1420 :
1421 : #ifndef arch_atomic64_fetch_add_acquire
1422 : static __always_inline s64
1423 : arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1424 : {
1425 : s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1426 : __atomic_acquire_fence();
1427 : return ret;
1428 : }
1429 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1430 : #endif
1431 :
1432 : #ifndef arch_atomic64_fetch_add_release
1433 : static __always_inline s64
1434 : arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1435 : {
1436 : __atomic_release_fence();
1437 : return arch_atomic64_fetch_add_relaxed(i, v);
1438 : }
1439 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1440 : #endif
1441 :
1442 : #ifndef arch_atomic64_fetch_add
1443 : static __always_inline s64
1444 : arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1445 : {
1446 : s64 ret;
1447 : __atomic_pre_full_fence();
1448 : ret = arch_atomic64_fetch_add_relaxed(i, v);
1449 : __atomic_post_full_fence();
1450 : return ret;
1451 : }
1452 : #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1453 : #endif
1454 :
1455 : #endif /* arch_atomic64_fetch_add_relaxed */
1456 :
1457 : #ifndef arch_atomic64_sub_return_relaxed
1458 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1459 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1460 : #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1461 : #else /* arch_atomic64_sub_return_relaxed */
1462 :
1463 : #ifndef arch_atomic64_sub_return_acquire
1464 : static __always_inline s64
1465 : arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1466 : {
1467 : s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1468 : __atomic_acquire_fence();
1469 : return ret;
1470 : }
1471 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1472 : #endif
1473 :
1474 : #ifndef arch_atomic64_sub_return_release
1475 : static __always_inline s64
1476 : arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1477 : {
1478 : __atomic_release_fence();
1479 : return arch_atomic64_sub_return_relaxed(i, v);
1480 : }
1481 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1482 : #endif
1483 :
1484 : #ifndef arch_atomic64_sub_return
1485 : static __always_inline s64
1486 : arch_atomic64_sub_return(s64 i, atomic64_t *v)
1487 : {
1488 : s64 ret;
1489 : __atomic_pre_full_fence();
1490 : ret = arch_atomic64_sub_return_relaxed(i, v);
1491 : __atomic_post_full_fence();
1492 : return ret;
1493 : }
1494 : #define arch_atomic64_sub_return arch_atomic64_sub_return
1495 : #endif
1496 :
1497 : #endif /* arch_atomic64_sub_return_relaxed */
1498 :
1499 : #ifndef arch_atomic64_fetch_sub_relaxed
1500 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1501 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1502 : #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1503 : #else /* arch_atomic64_fetch_sub_relaxed */
1504 :
1505 : #ifndef arch_atomic64_fetch_sub_acquire
1506 : static __always_inline s64
1507 : arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1508 : {
1509 : s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1510 : __atomic_acquire_fence();
1511 : return ret;
1512 : }
1513 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1514 : #endif
1515 :
1516 : #ifndef arch_atomic64_fetch_sub_release
1517 : static __always_inline s64
1518 : arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1519 : {
1520 : __atomic_release_fence();
1521 : return arch_atomic64_fetch_sub_relaxed(i, v);
1522 : }
1523 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1524 : #endif
1525 :
1526 : #ifndef arch_atomic64_fetch_sub
1527 : static __always_inline s64
1528 : arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1529 : {
1530 : s64 ret;
1531 : __atomic_pre_full_fence();
1532 : ret = arch_atomic64_fetch_sub_relaxed(i, v);
1533 : __atomic_post_full_fence();
1534 : return ret;
1535 : }
1536 : #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1537 : #endif
1538 :
1539 : #endif /* arch_atomic64_fetch_sub_relaxed */
1540 :
1541 : #ifndef arch_atomic64_inc
1542 : static __always_inline void
1543 : arch_atomic64_inc(atomic64_t *v)
1544 : {
1545 : arch_atomic64_add(1, v);
1546 : }
1547 : #define arch_atomic64_inc arch_atomic64_inc
1548 : #endif
1549 :
1550 : #ifndef arch_atomic64_inc_return_relaxed
1551 : #ifdef arch_atomic64_inc_return
1552 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1553 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1554 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1555 : #endif /* arch_atomic64_inc_return */
1556 :
1557 : #ifndef arch_atomic64_inc_return
1558 : static __always_inline s64
1559 : arch_atomic64_inc_return(atomic64_t *v)
1560 : {
1561 0 : return arch_atomic64_add_return(1, v);
1562 : }
1563 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1564 : #endif
1565 :
1566 : #ifndef arch_atomic64_inc_return_acquire
1567 : static __always_inline s64
1568 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1569 : {
1570 : return arch_atomic64_add_return_acquire(1, v);
1571 : }
1572 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1573 : #endif
1574 :
1575 : #ifndef arch_atomic64_inc_return_release
1576 : static __always_inline s64
1577 : arch_atomic64_inc_return_release(atomic64_t *v)
1578 : {
1579 : return arch_atomic64_add_return_release(1, v);
1580 : }
1581 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1582 : #endif
1583 :
1584 : #ifndef arch_atomic64_inc_return_relaxed
1585 : static __always_inline s64
1586 : arch_atomic64_inc_return_relaxed(atomic64_t *v)
1587 : {
1588 5 : return arch_atomic64_add_return_relaxed(1, v);
1589 : }
1590 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1591 : #endif
1592 :
1593 : #else /* arch_atomic64_inc_return_relaxed */
1594 :
1595 : #ifndef arch_atomic64_inc_return_acquire
1596 : static __always_inline s64
1597 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1598 : {
1599 : s64 ret = arch_atomic64_inc_return_relaxed(v);
1600 : __atomic_acquire_fence();
1601 : return ret;
1602 : }
1603 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1604 : #endif
1605 :
1606 : #ifndef arch_atomic64_inc_return_release
1607 : static __always_inline s64
1608 : arch_atomic64_inc_return_release(atomic64_t *v)
1609 : {
1610 : __atomic_release_fence();
1611 : return arch_atomic64_inc_return_relaxed(v);
1612 : }
1613 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1614 : #endif
1615 :
1616 : #ifndef arch_atomic64_inc_return
1617 : static __always_inline s64
1618 : arch_atomic64_inc_return(atomic64_t *v)
1619 : {
1620 : s64 ret;
1621 : __atomic_pre_full_fence();
1622 : ret = arch_atomic64_inc_return_relaxed(v);
1623 : __atomic_post_full_fence();
1624 : return ret;
1625 : }
1626 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1627 : #endif
1628 :
1629 : #endif /* arch_atomic64_inc_return_relaxed */
1630 :
1631 : #ifndef arch_atomic64_fetch_inc_relaxed
1632 : #ifdef arch_atomic64_fetch_inc
1633 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1634 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1635 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1636 : #endif /* arch_atomic64_fetch_inc */
1637 :
1638 : #ifndef arch_atomic64_fetch_inc
1639 : static __always_inline s64
1640 : arch_atomic64_fetch_inc(atomic64_t *v)
1641 : {
1642 : return arch_atomic64_fetch_add(1, v);
1643 : }
1644 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1645 : #endif
1646 :
1647 : #ifndef arch_atomic64_fetch_inc_acquire
1648 : static __always_inline s64
1649 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1650 : {
1651 : return arch_atomic64_fetch_add_acquire(1, v);
1652 : }
1653 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1654 : #endif
1655 :
1656 : #ifndef arch_atomic64_fetch_inc_release
1657 : static __always_inline s64
1658 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1659 : {
1660 : return arch_atomic64_fetch_add_release(1, v);
1661 : }
1662 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1663 : #endif
1664 :
1665 : #ifndef arch_atomic64_fetch_inc_relaxed
1666 : static __always_inline s64
1667 : arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1668 : {
1669 : return arch_atomic64_fetch_add_relaxed(1, v);
1670 : }
1671 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1672 : #endif
1673 :
1674 : #else /* arch_atomic64_fetch_inc_relaxed */
1675 :
1676 : #ifndef arch_atomic64_fetch_inc_acquire
1677 : static __always_inline s64
1678 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1679 : {
1680 : s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1681 : __atomic_acquire_fence();
1682 : return ret;
1683 : }
1684 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1685 : #endif
1686 :
1687 : #ifndef arch_atomic64_fetch_inc_release
1688 : static __always_inline s64
1689 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1690 : {
1691 : __atomic_release_fence();
1692 : return arch_atomic64_fetch_inc_relaxed(v);
1693 : }
1694 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1695 : #endif
1696 :
1697 : #ifndef arch_atomic64_fetch_inc
1698 : static __always_inline s64
1699 : arch_atomic64_fetch_inc(atomic64_t *v)
1700 : {
1701 : s64 ret;
1702 : __atomic_pre_full_fence();
1703 : ret = arch_atomic64_fetch_inc_relaxed(v);
1704 : __atomic_post_full_fence();
1705 : return ret;
1706 : }
1707 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1708 : #endif
1709 :
1710 : #endif /* arch_atomic64_fetch_inc_relaxed */
1711 :
1712 : #ifndef arch_atomic64_dec
1713 : static __always_inline void
1714 : arch_atomic64_dec(atomic64_t *v)
1715 : {
1716 : arch_atomic64_sub(1, v);
1717 : }
1718 : #define arch_atomic64_dec arch_atomic64_dec
1719 : #endif
1720 :
1721 : #ifndef arch_atomic64_dec_return_relaxed
1722 : #ifdef arch_atomic64_dec_return
1723 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1724 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1725 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1726 : #endif /* arch_atomic64_dec_return */
1727 :
1728 : #ifndef arch_atomic64_dec_return
1729 : static __always_inline s64
1730 : arch_atomic64_dec_return(atomic64_t *v)
1731 : {
1732 : return arch_atomic64_sub_return(1, v);
1733 : }
1734 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1735 : #endif
1736 :
1737 : #ifndef arch_atomic64_dec_return_acquire
1738 : static __always_inline s64
1739 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1740 : {
1741 : return arch_atomic64_sub_return_acquire(1, v);
1742 : }
1743 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1744 : #endif
1745 :
1746 : #ifndef arch_atomic64_dec_return_release
1747 : static __always_inline s64
1748 : arch_atomic64_dec_return_release(atomic64_t *v)
1749 : {
1750 : return arch_atomic64_sub_return_release(1, v);
1751 : }
1752 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1753 : #endif
1754 :
1755 : #ifndef arch_atomic64_dec_return_relaxed
1756 : static __always_inline s64
1757 : arch_atomic64_dec_return_relaxed(atomic64_t *v)
1758 : {
1759 : return arch_atomic64_sub_return_relaxed(1, v);
1760 : }
1761 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1762 : #endif
1763 :
1764 : #else /* arch_atomic64_dec_return_relaxed */
1765 :
1766 : #ifndef arch_atomic64_dec_return_acquire
1767 : static __always_inline s64
1768 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1769 : {
1770 : s64 ret = arch_atomic64_dec_return_relaxed(v);
1771 : __atomic_acquire_fence();
1772 : return ret;
1773 : }
1774 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1775 : #endif
1776 :
1777 : #ifndef arch_atomic64_dec_return_release
1778 : static __always_inline s64
1779 : arch_atomic64_dec_return_release(atomic64_t *v)
1780 : {
1781 : __atomic_release_fence();
1782 : return arch_atomic64_dec_return_relaxed(v);
1783 : }
1784 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1785 : #endif
1786 :
1787 : #ifndef arch_atomic64_dec_return
1788 : static __always_inline s64
1789 : arch_atomic64_dec_return(atomic64_t *v)
1790 : {
1791 : s64 ret;
1792 : __atomic_pre_full_fence();
1793 : ret = arch_atomic64_dec_return_relaxed(v);
1794 : __atomic_post_full_fence();
1795 : return ret;
1796 : }
1797 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1798 : #endif
1799 :
1800 : #endif /* arch_atomic64_dec_return_relaxed */
1801 :
1802 : #ifndef arch_atomic64_fetch_dec_relaxed
1803 : #ifdef arch_atomic64_fetch_dec
1804 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1805 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1806 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1807 : #endif /* arch_atomic64_fetch_dec */
1808 :
1809 : #ifndef arch_atomic64_fetch_dec
1810 : static __always_inline s64
1811 : arch_atomic64_fetch_dec(atomic64_t *v)
1812 : {
1813 : return arch_atomic64_fetch_sub(1, v);
1814 : }
1815 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1816 : #endif
1817 :
1818 : #ifndef arch_atomic64_fetch_dec_acquire
1819 : static __always_inline s64
1820 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1821 : {
1822 : return arch_atomic64_fetch_sub_acquire(1, v);
1823 : }
1824 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1825 : #endif
1826 :
1827 : #ifndef arch_atomic64_fetch_dec_release
1828 : static __always_inline s64
1829 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1830 : {
1831 : return arch_atomic64_fetch_sub_release(1, v);
1832 : }
1833 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1834 : #endif
1835 :
1836 : #ifndef arch_atomic64_fetch_dec_relaxed
1837 : static __always_inline s64
1838 : arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1839 : {
1840 : return arch_atomic64_fetch_sub_relaxed(1, v);
1841 : }
1842 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1843 : #endif
1844 :
1845 : #else /* arch_atomic64_fetch_dec_relaxed */
1846 :
1847 : #ifndef arch_atomic64_fetch_dec_acquire
1848 : static __always_inline s64
1849 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1850 : {
1851 : s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1852 : __atomic_acquire_fence();
1853 : return ret;
1854 : }
1855 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1856 : #endif
1857 :
1858 : #ifndef arch_atomic64_fetch_dec_release
1859 : static __always_inline s64
1860 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1861 : {
1862 : __atomic_release_fence();
1863 : return arch_atomic64_fetch_dec_relaxed(v);
1864 : }
1865 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1866 : #endif
1867 :
1868 : #ifndef arch_atomic64_fetch_dec
1869 : static __always_inline s64
1870 : arch_atomic64_fetch_dec(atomic64_t *v)
1871 : {
1872 : s64 ret;
1873 : __atomic_pre_full_fence();
1874 : ret = arch_atomic64_fetch_dec_relaxed(v);
1875 : __atomic_post_full_fence();
1876 : return ret;
1877 : }
1878 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1879 : #endif
1880 :
1881 : #endif /* arch_atomic64_fetch_dec_relaxed */
1882 :
1883 : #ifndef arch_atomic64_fetch_and_relaxed
1884 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1885 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1886 : #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1887 : #else /* arch_atomic64_fetch_and_relaxed */
1888 :
1889 : #ifndef arch_atomic64_fetch_and_acquire
1890 : static __always_inline s64
1891 : arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1892 : {
1893 : s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1894 : __atomic_acquire_fence();
1895 : return ret;
1896 : }
1897 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1898 : #endif
1899 :
1900 : #ifndef arch_atomic64_fetch_and_release
1901 : static __always_inline s64
1902 : arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1903 : {
1904 : __atomic_release_fence();
1905 : return arch_atomic64_fetch_and_relaxed(i, v);
1906 : }
1907 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1908 : #endif
1909 :
1910 : #ifndef arch_atomic64_fetch_and
1911 : static __always_inline s64
1912 : arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1913 : {
1914 : s64 ret;
1915 : __atomic_pre_full_fence();
1916 : ret = arch_atomic64_fetch_and_relaxed(i, v);
1917 : __atomic_post_full_fence();
1918 : return ret;
1919 : }
1920 : #define arch_atomic64_fetch_and arch_atomic64_fetch_and
1921 : #endif
1922 :
1923 : #endif /* arch_atomic64_fetch_and_relaxed */
1924 :
1925 : #ifndef arch_atomic64_andnot
1926 : static __always_inline void
1927 : arch_atomic64_andnot(s64 i, atomic64_t *v)
1928 : {
1929 0 : arch_atomic64_and(~i, v);
1930 : }
1931 : #define arch_atomic64_andnot arch_atomic64_andnot
1932 : #endif
1933 :
1934 : #ifndef arch_atomic64_fetch_andnot_relaxed
1935 : #ifdef arch_atomic64_fetch_andnot
1936 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1937 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1938 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1939 : #endif /* arch_atomic64_fetch_andnot */
1940 :
1941 : #ifndef arch_atomic64_fetch_andnot
1942 : static __always_inline s64
1943 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1944 : {
1945 : return arch_atomic64_fetch_and(~i, v);
1946 : }
1947 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1948 : #endif
1949 :
1950 : #ifndef arch_atomic64_fetch_andnot_acquire
1951 : static __always_inline s64
1952 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1953 : {
1954 : return arch_atomic64_fetch_and_acquire(~i, v);
1955 : }
1956 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1957 : #endif
1958 :
1959 : #ifndef arch_atomic64_fetch_andnot_release
1960 : static __always_inline s64
1961 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1962 : {
1963 : return arch_atomic64_fetch_and_release(~i, v);
1964 : }
1965 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1966 : #endif
1967 :
1968 : #ifndef arch_atomic64_fetch_andnot_relaxed
1969 : static __always_inline s64
1970 : arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1971 : {
1972 : return arch_atomic64_fetch_and_relaxed(~i, v);
1973 : }
1974 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1975 : #endif
1976 :
1977 : #else /* arch_atomic64_fetch_andnot_relaxed */
1978 :
1979 : #ifndef arch_atomic64_fetch_andnot_acquire
1980 : static __always_inline s64
1981 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1982 : {
1983 : s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1984 : __atomic_acquire_fence();
1985 : return ret;
1986 : }
1987 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1988 : #endif
1989 :
1990 : #ifndef arch_atomic64_fetch_andnot_release
1991 : static __always_inline s64
1992 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1993 : {
1994 : __atomic_release_fence();
1995 : return arch_atomic64_fetch_andnot_relaxed(i, v);
1996 : }
1997 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1998 : #endif
1999 :
2000 : #ifndef arch_atomic64_fetch_andnot
2001 : static __always_inline s64
2002 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
2003 : {
2004 : s64 ret;
2005 : __atomic_pre_full_fence();
2006 : ret = arch_atomic64_fetch_andnot_relaxed(i, v);
2007 : __atomic_post_full_fence();
2008 : return ret;
2009 : }
2010 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
2011 : #endif
2012 :
2013 : #endif /* arch_atomic64_fetch_andnot_relaxed */
2014 :
2015 : #ifndef arch_atomic64_fetch_or_relaxed
2016 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
2017 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
2018 : #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
2019 : #else /* arch_atomic64_fetch_or_relaxed */
2020 :
2021 : #ifndef arch_atomic64_fetch_or_acquire
2022 : static __always_inline s64
2023 : arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
2024 : {
2025 : s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
2026 : __atomic_acquire_fence();
2027 : return ret;
2028 : }
2029 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
2030 : #endif
2031 :
2032 : #ifndef arch_atomic64_fetch_or_release
2033 : static __always_inline s64
2034 : arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
2035 : {
2036 : __atomic_release_fence();
2037 : return arch_atomic64_fetch_or_relaxed(i, v);
2038 : }
2039 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
2040 : #endif
2041 :
2042 : #ifndef arch_atomic64_fetch_or
2043 : static __always_inline s64
2044 : arch_atomic64_fetch_or(s64 i, atomic64_t *v)
2045 : {
2046 : s64 ret;
2047 : __atomic_pre_full_fence();
2048 : ret = arch_atomic64_fetch_or_relaxed(i, v);
2049 : __atomic_post_full_fence();
2050 : return ret;
2051 : }
2052 : #define arch_atomic64_fetch_or arch_atomic64_fetch_or
2053 : #endif
2054 :
2055 : #endif /* arch_atomic64_fetch_or_relaxed */
2056 :
2057 : #ifndef arch_atomic64_fetch_xor_relaxed
2058 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
2059 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
2060 : #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
2061 : #else /* arch_atomic64_fetch_xor_relaxed */
2062 :
2063 : #ifndef arch_atomic64_fetch_xor_acquire
2064 : static __always_inline s64
2065 : arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
2066 : {
2067 : s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
2068 : __atomic_acquire_fence();
2069 : return ret;
2070 : }
2071 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
2072 : #endif
2073 :
2074 : #ifndef arch_atomic64_fetch_xor_release
2075 : static __always_inline s64
2076 : arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2077 : {
2078 : __atomic_release_fence();
2079 : return arch_atomic64_fetch_xor_relaxed(i, v);
2080 : }
2081 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
2082 : #endif
2083 :
2084 : #ifndef arch_atomic64_fetch_xor
2085 : static __always_inline s64
2086 : arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
2087 : {
2088 : s64 ret;
2089 : __atomic_pre_full_fence();
2090 : ret = arch_atomic64_fetch_xor_relaxed(i, v);
2091 : __atomic_post_full_fence();
2092 : return ret;
2093 : }
2094 : #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
2095 : #endif
2096 :
2097 : #endif /* arch_atomic64_fetch_xor_relaxed */
2098 :
2099 : #ifndef arch_atomic64_xchg_relaxed
2100 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg
2101 : #define arch_atomic64_xchg_release arch_atomic64_xchg
2102 : #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2103 : #else /* arch_atomic64_xchg_relaxed */
2104 :
2105 : #ifndef arch_atomic64_xchg_acquire
2106 : static __always_inline s64
2107 : arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2108 : {
2109 : s64 ret = arch_atomic64_xchg_relaxed(v, i);
2110 : __atomic_acquire_fence();
2111 : return ret;
2112 : }
2113 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2114 : #endif
2115 :
2116 : #ifndef arch_atomic64_xchg_release
2117 : static __always_inline s64
2118 : arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2119 : {
2120 : __atomic_release_fence();
2121 : return arch_atomic64_xchg_relaxed(v, i);
2122 : }
2123 : #define arch_atomic64_xchg_release arch_atomic64_xchg_release
2124 : #endif
2125 :
2126 : #ifndef arch_atomic64_xchg
2127 : static __always_inline s64
2128 : arch_atomic64_xchg(atomic64_t *v, s64 i)
2129 : {
2130 : s64 ret;
2131 : __atomic_pre_full_fence();
2132 : ret = arch_atomic64_xchg_relaxed(v, i);
2133 : __atomic_post_full_fence();
2134 : return ret;
2135 : }
2136 : #define arch_atomic64_xchg arch_atomic64_xchg
2137 : #endif
2138 :
2139 : #endif /* arch_atomic64_xchg_relaxed */
2140 :
2141 : #ifndef arch_atomic64_cmpxchg_relaxed
2142 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2143 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2144 : #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2145 : #else /* arch_atomic64_cmpxchg_relaxed */
2146 :
2147 : #ifndef arch_atomic64_cmpxchg_acquire
2148 : static __always_inline s64
2149 : arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2150 : {
2151 : s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2152 : __atomic_acquire_fence();
2153 : return ret;
2154 : }
2155 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2156 : #endif
2157 :
2158 : #ifndef arch_atomic64_cmpxchg_release
2159 : static __always_inline s64
2160 : arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2161 : {
2162 : __atomic_release_fence();
2163 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
2164 : }
2165 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2166 : #endif
2167 :
2168 : #ifndef arch_atomic64_cmpxchg
2169 : static __always_inline s64
2170 : arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2171 : {
2172 : s64 ret;
2173 : __atomic_pre_full_fence();
2174 : ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2175 : __atomic_post_full_fence();
2176 : return ret;
2177 : }
2178 : #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2179 : #endif
2180 :
2181 : #endif /* arch_atomic64_cmpxchg_relaxed */
2182 :
2183 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2184 : #ifdef arch_atomic64_try_cmpxchg
2185 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2186 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2187 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2188 : #endif /* arch_atomic64_try_cmpxchg */
2189 :
2190 : #ifndef arch_atomic64_try_cmpxchg
2191 : static __always_inline bool
2192 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2193 : {
2194 : s64 r, o = *old;
2195 : r = arch_atomic64_cmpxchg(v, o, new);
2196 : if (unlikely(r != o))
2197 : *old = r;
2198 : return likely(r == o);
2199 : }
2200 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2201 : #endif
2202 :
2203 : #ifndef arch_atomic64_try_cmpxchg_acquire
2204 : static __always_inline bool
2205 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2206 : {
2207 : s64 r, o = *old;
2208 : r = arch_atomic64_cmpxchg_acquire(v, o, new);
2209 : if (unlikely(r != o))
2210 : *old = r;
2211 : return likely(r == o);
2212 : }
2213 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2214 : #endif
2215 :
2216 : #ifndef arch_atomic64_try_cmpxchg_release
2217 : static __always_inline bool
2218 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2219 : {
2220 : s64 r, o = *old;
2221 : r = arch_atomic64_cmpxchg_release(v, o, new);
2222 : if (unlikely(r != o))
2223 : *old = r;
2224 : return likely(r == o);
2225 : }
2226 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2227 : #endif
2228 :
2229 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2230 : static __always_inline bool
2231 : arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2232 : {
2233 : s64 r, o = *old;
2234 : r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2235 : if (unlikely(r != o))
2236 : *old = r;
2237 : return likely(r == o);
2238 : }
2239 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2240 : #endif
2241 :
2242 : #else /* arch_atomic64_try_cmpxchg_relaxed */
2243 :
2244 : #ifndef arch_atomic64_try_cmpxchg_acquire
2245 : static __always_inline bool
2246 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2247 : {
2248 : bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2249 : __atomic_acquire_fence();
2250 : return ret;
2251 : }
2252 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2253 : #endif
2254 :
2255 : #ifndef arch_atomic64_try_cmpxchg_release
2256 : static __always_inline bool
2257 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2258 : {
2259 : __atomic_release_fence();
2260 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2261 : }
2262 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2263 : #endif
2264 :
2265 : #ifndef arch_atomic64_try_cmpxchg
2266 : static __always_inline bool
2267 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2268 : {
2269 : bool ret;
2270 : __atomic_pre_full_fence();
2271 : ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2272 : __atomic_post_full_fence();
2273 : return ret;
2274 : }
2275 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2276 : #endif
2277 :
2278 : #endif /* arch_atomic64_try_cmpxchg_relaxed */
2279 :
2280 : #ifndef arch_atomic64_sub_and_test
2281 : /**
2282 : * arch_atomic64_sub_and_test - subtract value from variable and test result
2283 : * @i: integer value to subtract
2284 : * @v: pointer of type atomic64_t
2285 : *
2286 : * Atomically subtracts @i from @v and returns
2287 : * true if the result is zero, or false for all
2288 : * other cases.
2289 : */
2290 : static __always_inline bool
2291 : arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2292 : {
2293 : return arch_atomic64_sub_return(i, v) == 0;
2294 : }
2295 : #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2296 : #endif
2297 :
2298 : #ifndef arch_atomic64_dec_and_test
2299 : /**
2300 : * arch_atomic64_dec_and_test - decrement and test
2301 : * @v: pointer of type atomic64_t
2302 : *
2303 : * Atomically decrements @v by 1 and
2304 : * returns true if the result is 0, or false for all other
2305 : * cases.
2306 : */
2307 : static __always_inline bool
2308 : arch_atomic64_dec_and_test(atomic64_t *v)
2309 : {
2310 : return arch_atomic64_dec_return(v) == 0;
2311 : }
2312 : #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2313 : #endif
2314 :
2315 : #ifndef arch_atomic64_inc_and_test
2316 : /**
2317 : * arch_atomic64_inc_and_test - increment and test
2318 : * @v: pointer of type atomic64_t
2319 : *
2320 : * Atomically increments @v by 1
2321 : * and returns true if the result is zero, or false for all
2322 : * other cases.
2323 : */
2324 : static __always_inline bool
2325 : arch_atomic64_inc_and_test(atomic64_t *v)
2326 : {
2327 : return arch_atomic64_inc_return(v) == 0;
2328 : }
2329 : #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2330 : #endif
2331 :
2332 : #ifndef arch_atomic64_add_negative
2333 : /**
2334 : * arch_atomic64_add_negative - add and test if negative
2335 : * @i: integer value to add
2336 : * @v: pointer of type atomic64_t
2337 : *
2338 : * Atomically adds @i to @v and returns true
2339 : * if the result is negative, or false when
2340 : * result is greater than or equal to zero.
2341 : */
2342 : static __always_inline bool
2343 : arch_atomic64_add_negative(s64 i, atomic64_t *v)
2344 : {
2345 : return arch_atomic64_add_return(i, v) < 0;
2346 : }
2347 : #define arch_atomic64_add_negative arch_atomic64_add_negative
2348 : #endif
2349 :
2350 : #ifndef arch_atomic64_fetch_add_unless
2351 : /**
2352 : * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2353 : * @v: pointer of type atomic64_t
2354 : * @a: the amount to add to v...
2355 : * @u: ...unless v is equal to u.
2356 : *
2357 : * Atomically adds @a to @v, so long as @v was not already @u.
2358 : * Returns original value of @v
2359 : */
2360 : static __always_inline s64
2361 : arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2362 : {
2363 0 : s64 c = arch_atomic64_read(v);
2364 :
2365 : do {
2366 0 : if (unlikely(c == u))
2367 : break;
2368 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2369 :
2370 : return c;
2371 : }
2372 : #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2373 : #endif
2374 :
2375 : #ifndef arch_atomic64_add_unless
2376 : /**
2377 : * arch_atomic64_add_unless - add unless the number is already a given value
2378 : * @v: pointer of type atomic64_t
2379 : * @a: the amount to add to v...
2380 : * @u: ...unless v is equal to u.
2381 : *
2382 : * Atomically adds @a to @v, if @v was not already @u.
2383 : * Returns true if the addition was done.
2384 : */
2385 : static __always_inline bool
2386 : arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2387 : {
2388 0 : return arch_atomic64_fetch_add_unless(v, a, u) != u;
2389 : }
2390 : #define arch_atomic64_add_unless arch_atomic64_add_unless
2391 : #endif
2392 :
2393 : #ifndef arch_atomic64_inc_not_zero
2394 : /**
2395 : * arch_atomic64_inc_not_zero - increment unless the number is zero
2396 : * @v: pointer of type atomic64_t
2397 : *
2398 : * Atomically increments @v by 1, if @v is non-zero.
2399 : * Returns true if the increment was done.
2400 : */
2401 : static __always_inline bool
2402 : arch_atomic64_inc_not_zero(atomic64_t *v)
2403 : {
2404 0 : return arch_atomic64_add_unless(v, 1, 0);
2405 : }
2406 : #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2407 : #endif
2408 :
2409 : #ifndef arch_atomic64_inc_unless_negative
2410 : static __always_inline bool
2411 : arch_atomic64_inc_unless_negative(atomic64_t *v)
2412 : {
2413 : s64 c = arch_atomic64_read(v);
2414 :
2415 : do {
2416 : if (unlikely(c < 0))
2417 : return false;
2418 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2419 :
2420 : return true;
2421 : }
2422 : #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2423 : #endif
2424 :
2425 : #ifndef arch_atomic64_dec_unless_positive
2426 : static __always_inline bool
2427 : arch_atomic64_dec_unless_positive(atomic64_t *v)
2428 : {
2429 : s64 c = arch_atomic64_read(v);
2430 :
2431 : do {
2432 : if (unlikely(c > 0))
2433 : return false;
2434 : } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2435 :
2436 : return true;
2437 : }
2438 : #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2439 : #endif
2440 :
2441 : #ifndef arch_atomic64_dec_if_positive
2442 : static __always_inline s64
2443 : arch_atomic64_dec_if_positive(atomic64_t *v)
2444 : {
2445 0 : s64 dec, c = arch_atomic64_read(v);
2446 :
2447 : do {
2448 0 : dec = c - 1;
2449 0 : if (unlikely(dec < 0))
2450 : break;
2451 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2452 :
2453 : return dec;
2454 : }
2455 : #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2456 : #endif
2457 :
2458 : #endif /* _LINUX_ATOMIC_FALLBACK_H */
2459 : // b5e87bdd5ede61470c29f7a7e4de781af3770f09
|
