Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _ASM_X86_ATOMIC64_64_H
       3             : #define _ASM_X86_ATOMIC64_64_H
       4             : 
       5             : #include <linux/types.h>
       6             : #include <asm/alternative.h>
       7             : #include <asm/cmpxchg.h>
       8             : 
       9             : /* The 64-bit atomic type */
      10             : 
      11             : #define ATOMIC64_INIT(i)        { (i) }
      12             : 
      13             : /**
      14             :  * arch_atomic64_read - read atomic64 variable
      15             :  * @v: pointer of type atomic64_t
      16             :  *
      17             :  * Atomically reads the value of @v.
      18             :  * Doesn't imply a read memory barrier.
      19             :  */
      20             : static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
      21             : {
      22       32814 :         return __READ_ONCE((v)->counter);
      23             : }
      24             : 
      25             : /**
      26             :  * arch_atomic64_set - set atomic64 variable
      27             :  * @v: pointer to type atomic64_t
      28             :  * @i: required value
      29             :  *
      30             :  * Atomically sets the value of @v to @i.
      31             :  */
      32             : static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
      33             : {
      34       69889 :         __WRITE_ONCE(v->counter, i);
      35             : }
      36             : 
      37             : /**
      38             :  * arch_atomic64_add - add integer to atomic64 variable
      39             :  * @i: integer value to add
      40             :  * @v: pointer to type atomic64_t
      41             :  *
      42             :  * Atomically adds @i to @v.
      43             :  */
      44             : static __always_inline void arch_atomic64_add(s64 i, atomic64_t *v)
      45             : {
      46       19159 :         asm volatile(LOCK_PREFIX "addq %1,%0"
      47             :                      : "=m" (v->counter)
      48             :                      : "er" (i), "m" (v->counter) : "memory");
      49             : }
      50             : 
      51             : /**
      52             :  * arch_atomic64_sub - subtract the atomic64 variable
      53             :  * @i: integer value to subtract
      54             :  * @v: pointer to type atomic64_t
      55             :  *
      56             :  * Atomically subtracts @i from @v.
      57             :  */
      58             : static __always_inline void arch_atomic64_sub(s64 i, atomic64_t *v)
      59             : {
      60        2016 :         asm volatile(LOCK_PREFIX "subq %1,%0"
      61             :                      : "=m" (v->counter)
      62             :                      : "er" (i), "m" (v->counter) : "memory");
      63             : }
      64             : 
      65             : /**
      66             :  * arch_atomic64_sub_and_test - subtract value from variable and test result
      67             :  * @i: integer value to subtract
      68             :  * @v: pointer to type atomic64_t
      69             :  *
      70             :  * Atomically subtracts @i from @v and returns
      71             :  * true if the result is zero, or false for all
      72             :  * other cases.
      73             :  */
      74             : static __always_inline bool arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
      75             : {
      76           0 :         return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
      77             : }
      78             : #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
      79             : 
      80             : /**
      81             :  * arch_atomic64_inc - increment atomic64 variable
      82             :  * @v: pointer to type atomic64_t
      83             :  *
      84             :  * Atomically increments @v by 1.
      85             :  */
      86             : static __always_inline void arch_atomic64_inc(atomic64_t *v)
      87             : {
      88        2194 :         asm volatile(LOCK_PREFIX "incq %0"
      89             :                      : "=m" (v->counter)
      90             :                      : "m" (v->counter) : "memory");
      91             : }
      92             : #define arch_atomic64_inc arch_atomic64_inc
      93             : 
      94             : /**
      95             :  * arch_atomic64_dec - decrement atomic64 variable
      96             :  * @v: pointer to type atomic64_t
      97             :  *
      98             :  * Atomically decrements @v by 1.
      99             :  */
     100             : static __always_inline void arch_atomic64_dec(atomic64_t *v)
     101             : {
     102        1753 :         asm volatile(LOCK_PREFIX "decq %0"
     103             :                      : "=m" (v->counter)
     104             :                      : "m" (v->counter) : "memory");
     105             : }
     106             : #define arch_atomic64_dec arch_atomic64_dec
     107             : 
     108             : /**
     109             :  * arch_atomic64_dec_and_test - decrement and test
     110             :  * @v: pointer to type atomic64_t
     111             :  *
     112             :  * Atomically decrements @v by 1 and
     113             :  * returns true if the result is 0, or false for all other
     114             :  * cases.
     115             :  */
     116             : static __always_inline bool arch_atomic64_dec_and_test(atomic64_t *v)
     117             : {
     118           0 :         return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
     119             : }
     120             : #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
     121             : 
     122             : /**
     123             :  * arch_atomic64_inc_and_test - increment and test
     124             :  * @v: pointer to type atomic64_t
     125             :  *
     126             :  * Atomically increments @v by 1
     127             :  * and returns true if the result is zero, or false for all
     128             :  * other cases.
     129             :  */
     130             : static __always_inline bool arch_atomic64_inc_and_test(atomic64_t *v)
     131             : {
     132             :         return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
     133             : }
     134             : #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
     135             : 
     136             : /**
     137             :  * arch_atomic64_add_negative - add and test if negative
     138             :  * @i: integer value to add
     139             :  * @v: pointer to type atomic64_t
     140             :  *
     141             :  * Atomically adds @i to @v and returns true
     142             :  * if the result is negative, or false when
     143             :  * result is greater than or equal to zero.
     144             :  */
     145             : static __always_inline bool arch_atomic64_add_negative(s64 i, atomic64_t *v)
     146             : {
     147             :         return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
     148             : }
     149             : #define arch_atomic64_add_negative arch_atomic64_add_negative
     150             : 
     151             : /**
     152             :  * arch_atomic64_add_return - add and return
     153             :  * @i: integer value to add
     154             :  * @v: pointer to type atomic64_t
     155             :  *
     156             :  * Atomically adds @i to @v and returns @i + @v
     157             :  */
     158             : static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v)
     159             : {
     160        2481 :         return i + xadd(&v->counter, i);
     161             : }
     162             : #define arch_atomic64_add_return arch_atomic64_add_return
     163             : 
     164             : static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v)
     165             : {
     166         734 :         return arch_atomic64_add_return(-i, v);
     167             : }
     168             : #define arch_atomic64_sub_return arch_atomic64_sub_return
     169             : 
     170             : static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
     171             : {
     172       27889 :         return xadd(&v->counter, i);
     173             : }
     174             : #define arch_atomic64_fetch_add arch_atomic64_fetch_add
     175             : 
     176             : static __always_inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
     177             : {
     178             :         return xadd(&v->counter, -i);
     179             : }
     180             : #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
     181             : 
     182             : static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
     183             : {
     184         901 :         return arch_cmpxchg(&v->counter, old, new);
     185             : }
     186             : #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
     187             : 
     188             : static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
     189             : {
     190       43521 :         return arch_try_cmpxchg(&v->counter, old, new);
     191             : }
     192             : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
     193             : 
     194             : static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new)
     195             : {
     196           0 :         return arch_xchg(&v->counter, new);
     197             : }
     198             : #define arch_atomic64_xchg arch_atomic64_xchg
     199             : 
     200             : static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v)
     201             : {
     202           0 :         asm volatile(LOCK_PREFIX "andq %1,%0"
     203             :                         : "+m" (v->counter)
     204             :                         : "er" (i)
     205             :                         : "memory");
     206             : }
     207             : 
     208             : static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
     209             : {
     210             :         s64 val = arch_atomic64_read(v);
     211             : 
     212             :         do {
     213             :         } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
     214             :         return val;
     215             : }
     216             : #define arch_atomic64_fetch_and arch_atomic64_fetch_and
     217             : 
     218             : static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v)
     219             : {
     220           0 :         asm volatile(LOCK_PREFIX "orq %1,%0"
     221             :                         : "+m" (v->counter)
     222             :                         : "er" (i)
     223             :                         : "memory");
     224             : }
     225             : 
     226             : static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
     227             : {
     228             :         s64 val = arch_atomic64_read(v);
     229             : 
     230             :         do {
     231             :         } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
     232             :         return val;
     233             : }
     234             : #define arch_atomic64_fetch_or arch_atomic64_fetch_or
     235             : 
     236             : static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v)
     237             : {
     238             :         asm volatile(LOCK_PREFIX "xorq %1,%0"
     239             :                         : "+m" (v->counter)
     240             :                         : "er" (i)
     241             :                         : "memory");
     242             : }
     243             : 
     244             : static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
     245             : {
     246             :         s64 val = arch_atomic64_read(v);
     247             : 
     248             :         do {
     249             :         } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
     250             :         return val;
     251             : }
     252             : #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
     253             : 
     254             : #endif /* _ASM_X86_ATOMIC64_64_H */