/* SPDX-License-Identifier: GPL-2.0-or-later
* Copyright (c) 2016 Cyril Hrubis <chrubis@suse.cz>
*/
/* The LTP library has some of its own atomic synchronisation primitives
* contained in this file. Generally speaking these should not be used
* directly in tests for synchronisation, instead use tst_checkpoint.h,
* tst_fuzzy_sync.h or the POSIX library.
*
* Notes on compile and runtime memory barriers and atomics.
*
* Within the LTP library we have three concerns when accessing variables
* shared by multiple threads or processes:
*
* (1) Removal or reordering of accesses by the compiler.
* (2) Atomicity of addition.
* (3) LOAD-STORE ordering between threads.
*
* The first (1) is the most likely to cause an error if not properly
* handled. We avoid it by using volatile variables and statements which will
* not be removed or reordered by the compiler during optimisation. This includes
* the __atomic and __sync intrinsics and volatile asm statements marked with
* "memory" as well as variables marked with volatile.
*
* On any platform Linux is likely to run on, a LOAD (fetch) or STORE of a
* 32-bit integer will be atomic. However fetching and adding to a variable is
* quite likely not; so for (2) we need to ensure we use atomic addition.
*
* Finally, for tst_fuzzy_sync at least, we need to ensure that LOADs and
* STOREs of any shared variables (including non-atomics) that are made
* between calls to tst_fzsync_wait are completed (globally visible) before
* tst_fzsync_wait completes. For this, runtime memory and instruction
* barriers are required in addition to compile time.
*
* We use full sequential ordering (__ATOMIC_SEQ_CST) for the sake of
* simplicity. LTP tests tend to be syscall heavy so any performance gain from
* using a weaker memory model is unlikely to result in a relatively large
* performance improvement while at the same time being a potent source of
* confusion.
*
* Likewise, for the fallback ASM, the simplest "definitely will work, always"
* approach is preferred over anything more performant.
*
* Also see Documentation/memory-barriers.txt in the kernel tree and
* https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
* terminology may vary between sources.
*/
#ifndef TST_ATOMIC_H__
#define TST_ATOMIC_H__
#include "config.h"
#if HAVE_ATOMIC_MEMORY_MODEL == 1
static inline int tst_atomic_add_return(int i, int *v)
{
return __atomic_add_fetch(v, i, __ATOMIC_SEQ_CST);
}
static inline int tst_atomic_load(int *v)
{
return __atomic_load_n(v, __ATOMIC_SEQ_CST);
}
static inline void tst_atomic_store(int i, int *v)
{
__atomic_store_n(v, i, __ATOMIC_SEQ_CST);
}
#elif HAVE_SYNC_ADD_AND_FETCH == 1
static inline int tst_atomic_add_return(int i, int *v)
{
return __sync_add_and_fetch(v, i);
}
static inline int tst_atomic_load(int *v)
{
int ret;
__sync_synchronize();
ret = *v;
__sync_synchronize();
return ret;
}
static inline void tst_atomic_store(int i, int *v)
{
__sync_synchronize();
*v = i;
__sync_synchronize();
}
#elif defined(__i386__) || defined(__x86_64__)
# define LTP_USE_GENERIC_LOAD_STORE_ASM 1
static inline int tst_atomic_add_return(int i, int *v)
{
int __ret = i;
/*
* taken from arch/x86/include/asm/cmpxchg.h
*/
asm volatile ("lock; xaddl %0, %1\n"
: "+r" (__ret), "+m" (*v) : : "memory", "cc");
return i + __ret;
}
#elif defined(__powerpc__) || defined(__powerpc64__)
static inline int tst_atomic_add_return(int i, int *v)
{
int t;
/* taken from arch/powerpc/include/asm/atomic.h */
asm volatile(
" sync\n"
"1: lwarx %0,0,%2 # atomic_add_return\n"
" add %0,%1,%0\n"
" stwcx. %0,0,%2 \n"
" bne- 1b\n"
" sync\n"
: "=&r" (t)
: "r" (i), "r" (v)
: "cc", "memory");
return t;
}
static inline int tst_atomic_load(int *v)
{
int ret;
asm volatile("sync\n" : : : "memory");
ret = *v;
asm volatile("sync\n" : : : "memory");
return ret;
}
static inline void tst_atomic_store(int i, int *v)
{
asm volatile("sync\n" : : : "memory");
*v = i;
asm volatile("sync\n" : : : "memory");
}
#elif defined(__s390__) || defined(__s390x__)
# define LTP_USE_GENERIC_LOAD_STORE_ASM 1
static inline int tst_atomic_add_return(int i, int *v)
{
int old_val, new_val;
/* taken from arch/s390/include/asm/atomic.h */
asm volatile(
" l %0,%2\n"
"0: lr %1,%0\n"
" ar %1,%3\n"
" cs %0,%1,%2\n"
" jl 0b"
: "=&d" (old_val), "=&d" (new_val), "+Q" (*v)
: "d" (i)
: "cc", "memory");
return old_val + i;
}
#elif defined(__arc__)
/*ARCv2 defines the smp barriers */
#ifdef __ARC700__
#define smp_mb() asm volatile("" : : : "memory")
#else
#define smp_mb() asm volatile("dmb 3\n" : : : "memory")
#endif
static inline int tst_atomic_add_return(int i, int *v)
{
unsigned int val;
smp_mb();
asm volatile(
"1: llock %[val], [%[ctr]] \n"
" add %[val], %[val], %[i] \n"
" scond %[val], [%[ctr]] \n"
" bnz 1b \n"
: [val] "=&r" (val)
: [ctr] "r" (v),
[i] "ir" (i)
: "cc", "memory");
smp_mb();
return val;
}
static inline int tst_atomic_load(int *v)
{
int ret;
smp_mb();
ret = *v;
smp_mb();
return ret;
}
static inline void tst_atomic_store(int i, int *v)
{
smp_mb();
*v = i;
smp_mb();
}
#elif defined (__aarch64__)
static inline int tst_atomic_add_return(int i, int *v)
{
unsigned long tmp;
int result;
__asm__ __volatile__(
" prfm pstl1strm, %2 \n"
"1: ldaxr %w0, %2 \n"
" add %w0, %w0, %w3 \n"
" stlxr %w1, %w0, %2 \n"
" cbnz %w1, 1b \n"
" dmb ish \n"
: "=&r" (result), "=&r" (tmp), "+Q" (*v)
: "Ir" (i)
: "memory");
return result;
}
/* We are using load and store exclusive (ldaxr & stlxr) instructions to try
* and help prevent the tst_atomic_load and, more likely, tst_atomic_store
* functions from interfering with tst_atomic_add_return which takes advantage
* of exclusivity. It is not clear if this is a good idea or not, but does
* mean that all three functions are very similar.
*/
static inline int tst_atomic_load(int *v)
{
int ret;
unsigned long tmp;
asm volatile("//atomic_load \n"
" prfm pstl1strm, %[v] \n"
"1: ldaxr %w[ret], %[v] \n"
" stlxr %w[tmp], %w[ret], %[v] \n"
" cbnz %w[tmp], 1b \n"
" dmb ish \n"
: [tmp] "=&r" (tmp), [ret] "=&r" (ret), [v] "+Q" (*v)
: : "memory");
return ret;
}
static inline void tst_atomic_store(int i, int *v)
{
unsigned long tmp;
asm volatile("//atomic_store \n"
" prfm pstl1strm, %[v] \n"
"1: ldaxr %w[tmp], %[v] \n"
" stlxr %w[tmp], %w[i], %[v] \n"
" cbnz %w[tmp], 1b \n"
" dmb ish \n"
: [tmp] "=&r" (tmp), [v] "+Q" (*v)
: [i] "r" (i)
: "memory");
}
#elif defined(__sparc__) && defined(__arch64__)
# define LTP_USE_GENERIC_LOAD_STORE_ASM 1
static inline int tst_atomic_add_return(int i, int *v)
{
int ret, tmp;
/* Based on arch/sparc/lib/atomic_64.S with the exponential backoff
* function removed because we are unlikely to have a large (>= 16?)
* number of cores continuously trying to update one variable.
*/
asm volatile("/*atomic_add_return*/ \n"
"1: ldsw [%[v]], %[ret]; \n"
" add %[ret], %[i], %[tmp]; \n"
" cas [%[v]], %[ret], %[tmp]; \n"
" cmp %[ret], %[tmp]; \n"
" bne,pn %%icc, 1b; \n"
" nop; \n"
" add %[ret], %[i], %[ret]; \n"
: [ret] "=r&" (ret), [tmp] "=r&" (tmp)
: [i] "r" (i), [v] "r" (v)
: "memory", "cc");
return ret;
}
#else /* HAVE_SYNC_ADD_AND_FETCH == 1 */
# error Your compiler does not provide __atomic_add_fetch, __sync_add_and_fetch \
and an LTP implementation is missing for your architecture.
#endif
#ifdef LTP_USE_GENERIC_LOAD_STORE_ASM
static inline int tst_atomic_load(int *v)
{
int ret;
asm volatile("" : : : "memory");
ret = *v;
asm volatile("" : : : "memory");
return ret;
}
static inline void tst_atomic_store(int i, int *v)
{
asm volatile("" : : : "memory");
*v = i;
asm volatile("" : : : "memory");
}
#endif
static inline int tst_atomic_inc(int *v)
{
return tst_atomic_add_return(1, v);
}
static inline int tst_atomic_dec(int *v)
{
return tst_atomic_add_return(-1, v);
}
#endif /* TST_ATOMIC_H__ */
