diff options
Diffstat (limited to 'include/haproxy/atomic.h')
| -rw-r--r-- | include/haproxy/atomic.h | 897 |
1 files changed, 897 insertions, 0 deletions
diff --git a/include/haproxy/atomic.h b/include/haproxy/atomic.h new file mode 100644 index 0000000..d64e192 --- /dev/null +++ b/include/haproxy/atomic.h @@ -0,0 +1,897 @@ +/* + * include/haproxy/atomic.h + * Macros and inline functions for thread-safe atomic operations. + * + * Copyright (C) 2017 Christopher Faulet - cfaulet@haproxy.com + * Copyright (C) 2020 Willy Tarreau - w@1wt.eu + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, version 2.1 + * exclusively. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _HAPROXY_ATOMIC_H +#define _HAPROXY_ATOMIC_H + +#include <haproxy/compiler.h> + +/* A few notes for the macros and functions here: + * - this file is painful to edit, most operations exist in 3 variants, + * no-thread, threads with gcc<4.7, threads with gcc>=4.7. Be careful when + * modifying it not to break any of them. + * + * - macros named HA_ATOMIC_* are or use in the general case, they contain the + * required memory barriers to guarantee sequential consistency + * + * - macros named _HA_ATOMIC_* are the same but without the memory barriers, + * so they may only be used if followed by other HA_ATOMIC_* or within a + * sequence of _HA_ATOMIC_* terminated by a store barrier, or when there is + * no data dependency (e.g. updating a counter). Not all of them are + * implemented, in which case fallbacks to the safe ones are provided. In + * case of doubt, don't use them and use the generic ones instead. + * + * - the __ha_atomic_* barriers are for use around _HA_ATOMIC_* operations. + * Some architectures make them useless and they will automatically be + * dropped in such a case. Don't use them outside of this use case. + * + * - in general, the more underscores you find in front of a function or macro + * name, the riskier it is to use. Barriers are among them because validating + * their usage is not trivial at all and it's often safer to fall back to + * more generic behaviors. + * + * There is also a compiler barrier (__ha_compiler_barrier) which is eliminated + * when threads are disabled. We currently don't have a permanent compiler + * barrier to prevent the compiler from reordering signal-sensitive code for + * example. + */ + + +#ifndef USE_THREAD + +/* Threads are DISABLED, atomic ops are also not used. Note that these MUST + * NOT be used for inter-process synchronization nor signal-safe variable + * manipulations which might occur without threads, as they are not atomic. + */ + +#define HA_ATOMIC_LOAD(val) *(val) +#define HA_ATOMIC_STORE(val, new) ({*(val) = new;}) + +#define HA_ATOMIC_XCHG(val, new) \ + ({ \ + typeof(*(val)) __old_xchg = *(val); \ + *(val) = new; \ + __old_xchg; \ + }) + +#define HA_ATOMIC_AND(val, flags) do { *(val) &= (flags);} while (0) +#define HA_ATOMIC_OR(val, flags) do { *(val) |= (flags);} while (0) +#define HA_ATOMIC_ADD(val, i) do { *(val) += (i);} while (0) +#define HA_ATOMIC_SUB(val, i) do { *(val) -= (i);} while (0) +#define HA_ATOMIC_INC(val) do { *(val) += 1;} while (0) +#define HA_ATOMIC_DEC(val) do { *(val) -= 1;} while (0) + +#define HA_ATOMIC_AND_FETCH(val, flags) ({ *(val) &= (flags); }) +#define HA_ATOMIC_OR_FETCH(val, flags) ({ *(val) |= (flags); }) +#define HA_ATOMIC_ADD_FETCH(val, i) ({ *(val) += (i); }) +#define HA_ATOMIC_SUB_FETCH(val, i) ({ *(val) -= (i); }) + +#define HA_ATOMIC_FETCH_AND(val, i) \ + ({ \ + typeof((val)) __p_val = (val); \ + typeof(*(val)) __old_val = *__p_val; \ + *__p_val &= (i); \ + __old_val; \ + }) + +#define HA_ATOMIC_FETCH_OR(val, i) \ + ({ \ + typeof((val)) __p_val = (val); \ + typeof(*(val)) __old_val = *__p_val; \ + *__p_val |= (i); \ + __old_val; \ + }) + +#define HA_ATOMIC_FETCH_ADD(val, i) \ + ({ \ + typeof((val)) __p_val = (val); \ + typeof(*(val)) __old_val = *__p_val; \ + *__p_val += (i); \ + __old_val; \ + }) + +#define HA_ATOMIC_FETCH_SUB(val, i) \ + ({ \ + typeof((val)) __p_val = (val); \ + typeof(*(val)) __old_val = *__p_val; \ + *__p_val -= (i); \ + __old_val; \ + }) + +#define HA_ATOMIC_BTS(val, bit) \ + ({ \ + typeof((val)) __p_bts = (val); \ + typeof(*__p_bts) __b_bts = (1UL << (bit)); \ + typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \ + if (!__t_bts) \ + *__p_bts |= __b_bts; \ + __t_bts; \ + }) + +#define HA_ATOMIC_BTR(val, bit) \ + ({ \ + typeof((val)) __p_btr = (val); \ + typeof(*__p_btr) __b_btr = (1UL << (bit)); \ + typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \ + if (__t_btr) \ + *__p_btr &= ~__b_btr; \ + __t_btr; \ + }) + +#define HA_ATOMIC_CAS(val, old, new) \ + ({ \ + typeof(val) _v = (val); \ + typeof(old) _o = (old); \ + (*_v == *_o) ? ((*_v = (new)), 1) : ((*_o = *_v), 0); \ + }) + +/* warning, n is a pointer to the double value for dwcas */ +#define HA_ATOMIC_DWCAS(val, o, n) \ + ({ \ + long *_v = (long*)(val); \ + long *_o = (long*)(o); \ + long *_n = (long*)(n); \ + long _v0 = _v[0], _v1 = _v[1]; \ + (_v0 == _o[0] && _v1 == _o[1]) ? \ + (_v[0] = _n[0], _v[1] = _n[1], 1) : \ + (_o[0] = _v0, _o[1] = _v1, 0); \ + }) + +#define HA_ATOMIC_UPDATE_MAX(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __new_max = (new); \ + \ + if (*__val < __new_max) \ + *__val = __new_max; \ + *__val; \ + }) + +#define HA_ATOMIC_UPDATE_MIN(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __new_min = (new); \ + \ + if (*__val > __new_min) \ + *__val = __new_min; \ + *__val; \ + }) + +/* various barriers */ +#define __ha_barrier_atomic_load() do { } while (0) +#define __ha_barrier_atomic_store() do { } while (0) +#define __ha_barrier_atomic_full() do { } while (0) +#define __ha_barrier_load() do { } while (0) +#define __ha_barrier_store() do { } while (0) +#define __ha_barrier_full() do { } while (0) +#define __ha_compiler_barrier() do { } while (0) +#define __ha_cpu_relax() ({ 1; }) + +#else /* !USE_THREAD */ + +/* Threads are ENABLED, all atomic ops are made thread-safe. By extension they + * can also be used for inter-process synchronization but one must verify that + * the code still builds with threads disabled. + */ + +#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__) +/* gcc < 4.7 */ + +#define HA_ATOMIC_LOAD(val) \ + ({ \ + typeof(*(val)) ret = \ + ({ __sync_synchronize(); *(volatile typeof(val))val; }); \ + __sync_synchronize(); \ + ret; \ + }) + +#define HA_ATOMIC_STORE(val, new) \ + ({ \ + typeof((val)) __val_store = (val); \ + typeof(*(val)) __old_store; \ + typeof((new)) __new_store = (new); \ + do { __old_store = *__val_store; \ + } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store) && __ha_cpu_relax()); \ + }) + +#define HA_ATOMIC_XCHG(val, new) \ + ({ \ + typeof((val)) __val_xchg = (val); \ + typeof(*(val)) __old_xchg; \ + typeof((new)) __new_xchg = (new); \ + do { __old_xchg = *__val_xchg; \ + } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg) && __ha_cpu_relax()); \ + __old_xchg; \ + }) + +#define HA_ATOMIC_AND(val, flags) do { __sync_and_and_fetch(val, flags); } while (0) +#define HA_ATOMIC_OR(val, flags) do { __sync_or_and_fetch(val, flags); } while (0) +#define HA_ATOMIC_ADD(val, i) do { __sync_add_and_fetch(val, i); } while (0) +#define HA_ATOMIC_SUB(val, i) do { __sync_sub_and_fetch(val, i); } while (0) +#define HA_ATOMIC_INC(val) do { __sync_add_and_fetch(val, 1); } while (0) +#define HA_ATOMIC_DEC(val) do { __sync_sub_and_fetch(val, 1); } while (0) + +#define HA_ATOMIC_AND_FETCH(val, flags) __sync_and_and_fetch(val, flags) +#define HA_ATOMIC_OR_FETCH(val, flags) __sync_or_and_fetch(val, flags) +#define HA_ATOMIC_ADD_FETCH(val, i) __sync_add_and_fetch(val, i) +#define HA_ATOMIC_SUB_FETCH(val, i) __sync_sub_and_fetch(val, i) + +#define HA_ATOMIC_FETCH_AND(val, flags) __sync_fetch_and_and(val, flags) +#define HA_ATOMIC_FETCH_OR(val, flags) __sync_fetch_and_or(val, flags) +#define HA_ATOMIC_FETCH_ADD(val, i) __sync_fetch_and_add(val, i) +#define HA_ATOMIC_FETCH_SUB(val, i) __sync_fetch_and_sub(val, i) + +#define HA_ATOMIC_BTS(val, bit) \ + ({ \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __sync_fetch_and_or((val), __b_bts) & __b_bts; \ + }) + +#define HA_ATOMIC_BTR(val, bit) \ + ({ \ + typeof(*(val)) __b_btr = (1UL << (bit)); \ + __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \ + }) + +/* the CAS is a bit complicated. The older API doesn't support returning the + * value and the swap's result at the same time. So here we take what looks + * like the safest route, consisting in using the boolean version guaranteeing + * that the operation was performed or not, and we snoop a previous value. If + * the compare succeeds, we return. If it fails, we return the previous value, + * but only if it differs from the expected one. If it's the same it's a race + * thus we try again to avoid confusing a possibly sensitive caller. + */ +#define HA_ATOMIC_CAS(val, old, new) \ + ({ \ + typeof((val)) __val_cas = (val); \ + typeof((old)) __oldp_cas = (old); \ + typeof(*(old)) __oldv_cas; \ + typeof((new)) __new_cas = (new); \ + int __ret_cas; \ + do { \ + __oldv_cas = *__val_cas; \ + __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \ + } while (!__ret_cas && *__oldp_cas == __oldv_cas && __ha_cpu_relax()); \ + if (!__ret_cas) \ + *__oldp_cas = __oldv_cas; \ + __ret_cas; \ + }) + +/* warning, n is a pointer to the double value for dwcas */ +#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n) + +#define HA_ATOMIC_UPDATE_MAX(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __old_max = *__val; \ + typeof(*(val)) __new_max = (new); \ + \ + while (__old_max < __new_max && \ + !HA_ATOMIC_CAS(__val, &__old_max, __new_max) && __ha_cpu_relax()); \ + *__val; \ + }) + +#define HA_ATOMIC_UPDATE_MIN(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __old_min = *__val; \ + typeof(*(val)) __new_min = (new); \ + \ + while (__old_min > __new_min && \ + !HA_ATOMIC_CAS(__val, &__old_min, __new_min) && __ha_cpu_relax()); \ + *__val; \ + }) + +#else /* gcc */ + +/* gcc >= 4.7 or clang */ + +#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_RELEASE) +#define HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_ACQUIRE) +#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_ACQ_REL) + +#define HA_ATOMIC_AND(val, flags) do { __atomic_and_fetch(val, flags, __ATOMIC_SEQ_CST); } while (0) +#define HA_ATOMIC_OR(val, flags) do { __atomic_or_fetch(val, flags, __ATOMIC_SEQ_CST); } while (0) +#define HA_ATOMIC_ADD(val, i) do { __atomic_add_fetch(val, i, __ATOMIC_SEQ_CST); } while (0) +#define HA_ATOMIC_SUB(val, i) do { __atomic_sub_fetch(val, i, __ATOMIC_SEQ_CST); } while (0) +#define HA_ATOMIC_INC(val) do { __atomic_add_fetch(val, 1, __ATOMIC_SEQ_CST); } while (0) +#define HA_ATOMIC_DEC(val) do { __atomic_sub_fetch(val, 1, __ATOMIC_SEQ_CST); } while (0) + +#define HA_ATOMIC_AND_FETCH(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_OR_FETCH(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_ADD_FETCH(val, i) __atomic_add_fetch(val, i, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_SUB_FETCH(val, i) __atomic_sub_fetch(val, i, __ATOMIC_SEQ_CST) + +#define HA_ATOMIC_FETCH_AND(val, flags) __atomic_fetch_and(val, flags, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_FETCH_OR(val, flags) __atomic_fetch_or(val, flags, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_FETCH_ADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_SEQ_CST) +#define HA_ATOMIC_FETCH_SUB(val, i) __atomic_fetch_sub(val, i, __ATOMIC_SEQ_CST) + +#if defined(__GCC_ASM_FLAG_OUTPUTS__) && (defined(__i386__) || defined (__x86_64__)) +#define HA_ATOMIC_BTS(val, bit) \ + ({ \ + unsigned char __ret; \ + if (sizeof(long) == 8 && sizeof(*(val)) == 8) { \ + asm volatile("lock btsq %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned long)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 4) { \ + asm volatile("lock btsl %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned int)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 2) { \ + asm volatile("lock btsw %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned short)(bit)) \ + : "cc"); \ + } else { \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __ret = !!(__atomic_fetch_or((val), __b_bts, __ATOMIC_SEQ_CST) & __b_bts); \ + } \ + __ret; \ + }) + +#define HA_ATOMIC_BTR(val, bit) \ + ({ \ + unsigned char __ret; \ + if (sizeof(long) == 8 && sizeof(*(val)) == 8) { \ + asm volatile("lock btrq %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned long)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 4) { \ + asm volatile("lock btrl %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned int)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 2) { \ + asm volatile("lock btrw %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned short)(bit)) \ + : "cc"); \ + } else { \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __ret = !!(__atomic_fetch_and((val), ~__b_bts, __ATOMIC_SEQ_CST) & __b_bts); \ + } \ + __ret; \ + }) + +#else // not x86 or !__GCC_ASM_FLAG_OUTPUTS__ + +#define HA_ATOMIC_BTS(val, bit) \ + ({ \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __atomic_fetch_or((val), __b_bts, __ATOMIC_SEQ_CST) & __b_bts; \ + }) + +#define HA_ATOMIC_BTR(val, bit) \ + ({ \ + typeof(*(val)) __b_btr = (1UL << (bit)); \ + __atomic_fetch_and((val), ~__b_btr, __ATOMIC_SEQ_CST) & __b_btr; \ + }) + +#endif // x86 || __GCC_ASM_FLAG_OUTPUTS__ + +#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) + +/* warning, n is a pointer to the double value for dwcas */ +#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n) + +#define HA_ATOMIC_UPDATE_MAX(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __old_max = *__val; \ + typeof(*(val)) __new_max = (new); \ + \ + while (__old_max < __new_max && \ + !HA_ATOMIC_CAS(__val, &__old_max, __new_max) && __ha_cpu_relax()); \ + *__val; \ + }) + +#define HA_ATOMIC_UPDATE_MIN(val, new) \ + ({ \ + typeof(val) __val = (val); \ + typeof(*(val)) __old_min = *__val; \ + typeof(*(val)) __new_min = (new); \ + \ + while (__old_min > __new_min && \ + !HA_ATOMIC_CAS(__val, &__old_min, __new_min) && __ha_cpu_relax()); \ + *__val; \ + }) + +/* Modern compilers provide variants that don't generate any memory barrier. + * If you're unsure how to deal with barriers, just use the HA_ATOMIC_* version, + * that will always generate correct code. + * Usually it's fine to use those when updating data that have no dependency, + * ie updating a counter. Otherwise a barrier is required. + */ + +#define _HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_RELAXED) +#define _HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_RELAXED) +#define _HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_RELAXED) + +#define _HA_ATOMIC_AND(val, flags) do { __atomic_and_fetch(val, flags, __ATOMIC_RELAXED); } while (0) +#define _HA_ATOMIC_OR(val, flags) do { __atomic_or_fetch(val, flags, __ATOMIC_RELAXED); } while (0) +#define _HA_ATOMIC_ADD(val, i) do { __atomic_add_fetch(val, i, __ATOMIC_RELAXED); } while (0) +#define _HA_ATOMIC_SUB(val, i) do { __atomic_sub_fetch(val, i, __ATOMIC_RELAXED); } while (0) +#define _HA_ATOMIC_INC(val) do { __atomic_add_fetch(val, 1, __ATOMIC_RELAXED); } while (0) +#define _HA_ATOMIC_DEC(val) do { __atomic_sub_fetch(val, 1, __ATOMIC_RELAXED); } while (0) + +#define _HA_ATOMIC_AND_FETCH(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_RELAXED) +#define _HA_ATOMIC_OR_FETCH(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_RELAXED) +#define _HA_ATOMIC_ADD_FETCH(val, i) __atomic_add_fetch(val, i, __ATOMIC_RELAXED) +#define _HA_ATOMIC_SUB_FETCH(val, i) __atomic_sub_fetch(val, i, __ATOMIC_RELAXED) + +#define _HA_ATOMIC_FETCH_AND(val, flags) __atomic_fetch_and(val, flags, __ATOMIC_RELAXED) +#define _HA_ATOMIC_FETCH_OR(val, flags) __atomic_fetch_or(val, flags, __ATOMIC_RELAXED) +#define _HA_ATOMIC_FETCH_ADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_RELAXED) +#define _HA_ATOMIC_FETCH_SUB(val, i) __atomic_fetch_sub(val, i, __ATOMIC_RELAXED) + +#if defined(__GCC_ASM_FLAG_OUTPUTS__) && (defined(__i386__) || defined (__x86_64__)) +#define _HA_ATOMIC_BTS(val, bit) \ + ({ \ + unsigned char __ret; \ + if (sizeof(long) == 8 && sizeof(*(val)) == 8) { \ + asm volatile("lock btsq %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned long)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 4) { \ + asm volatile("lock btsl %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned int)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 2) { \ + asm volatile("lock btsw %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned short)(bit)) \ + : "cc"); \ + } else { \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __ret = !!(__atomic_fetch_or((val), __b_bts, __ATOMIC_RELAXED) & __b_bts); \ + } \ + __ret; \ + }) + +#define _HA_ATOMIC_BTR(val, bit) \ + ({ \ + unsigned char __ret; \ + if (sizeof(long) == 8 && sizeof(*(val)) == 8) { \ + asm volatile("lock btrq %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned long)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 4) { \ + asm volatile("lock btrl %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned int)(bit)) \ + : "cc"); \ + } else if (sizeof(*(val)) == 2) { \ + asm volatile("lock btrw %2, %0\n" \ + : "+m" (*(val)), "=@ccc"(__ret) \ + : "Ir" ((unsigned short)(bit)) \ + : "cc"); \ + } else { \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __ret = !!(__atomic_fetch_and((val), ~__b_bts, __ATOMIC_RELAXED) & __b_bts); \ + } \ + __ret; \ + }) + +#else // not x86 or !__GCC_ASM_FLAG_OUTPUTS__ + +#define _HA_ATOMIC_BTS(val, bit) \ + ({ \ + typeof(*(val)) __b_bts = (1UL << (bit)); \ + __atomic_fetch_or((val), __b_bts, __ATOMIC_RELAXED) & __b_bts; \ + }) + +#define _HA_ATOMIC_BTR(val, bit) \ + ({ \ + typeof(*(val)) __b_btr = (1UL << (bit)); \ + __atomic_fetch_and((val), ~__b_btr, __ATOMIC_RELAXED) & __b_btr; \ + }) +#endif + +#define _HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED) +/* warning, n is a pointer to the double value for dwcas */ +#define _HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n) + +#endif /* gcc >= 4.7 */ + +/* Here come a few architecture-specific double-word CAS and barrier + * implementations. + */ + +#ifdef __x86_64__ + +static __inline void +__ha_barrier_load(void) +{ + __asm __volatile("" ::: "memory"); +} + +static __inline void +__ha_barrier_store(void) +{ + __asm __volatile("" ::: "memory"); +} + +static __inline void +__ha_barrier_full(void) +{ + __asm __volatile("mfence" ::: "memory"); +} + +/* Use __ha_barrier_atomic* when you're trying to protect data that are + * are modified using _HA_ATOMIC* + */ +static __inline void +__ha_barrier_atomic_load(void) +{ + __asm __volatile("" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_store(void) +{ + __asm __volatile("" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_full(void) +{ + __asm __volatile("" ::: "memory"); +} + +static __inline int +__ha_cas_dw(void *target, void *compare, const void *set) +{ + char ret; + + __asm __volatile("lock cmpxchg16b %0; setz %3" + : "+m" (*(void **)target), + "=a" (((void **)compare)[0]), + "=d" (((void **)compare)[1]), + "=q" (ret) + : "a" (((void **)compare)[0]), + "d" (((void **)compare)[1]), + "b" (((const void **)set)[0]), + "c" (((const void **)set)[1]) + : "memory", "cc"); + return (ret); +} + +/* short-lived CPU relaxation */ +#define __ha_cpu_relax() ({ asm volatile("rep;nop\n"); 1; }) + +#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)) + +static __inline void +__ha_barrier_load(void) +{ + __asm __volatile("dmb" ::: "memory"); +} + +static __inline void +__ha_barrier_store(void) +{ + __asm __volatile("dsb" ::: "memory"); +} + +static __inline void +__ha_barrier_full(void) +{ + __asm __volatile("dmb" ::: "memory"); +} + +/* Use __ha_barrier_atomic* when you're trying to protect data that are + * are modified using _HA_ATOMIC* + */ +static __inline void +__ha_barrier_atomic_load(void) +{ + __asm __volatile("dmb" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_store(void) +{ + __asm __volatile("dsb" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_full(void) +{ + __asm __volatile("dmb" ::: "memory"); +} + +static __inline int __ha_cas_dw(void *target, void *compare, const void *set) +{ + uint64_t previous; + int tmp; + + __asm __volatile("1:" + "ldrexd %0, [%4];" + "cmp %Q0, %Q2;" + "ittt eq;" + "cmpeq %R0, %R2;" + "strexdeq %1, %3, [%4];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), "=&r" (tmp) + : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target) + : "memory", "cc"); + tmp = (previous == *(uint64_t *)compare); + *(uint64_t *)compare = previous; + return (tmp); +} + +/* short-lived CPU relaxation */ +#define __ha_cpu_relax() ({ asm volatile(""); 1; }) + +#elif defined (__aarch64__) + +static __inline void +__ha_barrier_load(void) +{ + __asm __volatile("dmb ishld" ::: "memory"); +} + +static __inline void +__ha_barrier_store(void) +{ + __asm __volatile("dmb ishst" ::: "memory"); +} + +static __inline void +__ha_barrier_full(void) +{ + __asm __volatile("dmb ish" ::: "memory"); +} + +/* Use __ha_barrier_atomic* when you're trying to protect data that are + * are modified using _HA_ATOMIC* + */ +static __inline void +__ha_barrier_atomic_load(void) +{ + __asm __volatile("dmb ishld" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_store(void) +{ + __asm __volatile("dmb ishst" ::: "memory"); +} + +static __inline void +__ha_barrier_atomic_full(void) +{ + __asm __volatile("dmb ish" ::: "memory"); +} + +/* short-lived CPU relaxation; this was shown to improve fairness on + * modern ARMv8 cores such as Neoverse N1. + */ +#define __ha_cpu_relax() ({ asm volatile("isb" ::: "memory"); 1; }) + +#if defined(__ARM_FEATURE_ATOMICS) && !defined(__clang__) // ARMv8.1-A atomics + +/* returns 0 on failure, non-zero on success */ +static forceinline int __ha_cas_dw(void *target, void *compare, const void *set) +{ + /* There's no status set by the CASP instruction so we need to keep a + * copy of the original registers and compare them afterwards to detect + * if we could apply the change. In order to pass a pair, we simply map + * a register pair on a struct so that the compiler can emit register + * pairs that we can use thanks to the undocumented "%H" modifier + * mentioned on the link below: + * https://patchwork.ozlabs.org/project/gcc/patch/59368A74.2060908@foss.arm.com/ + */ + struct pair { uint64_t r[2]; }; + register struct pair bck = *(struct pair *)compare; + register struct pair cmp asm("x0") = bck; + register struct pair new asm("x2") = *(const struct pair*)set; + int ret; + + __asm__ __volatile__("casp %0, %H0, %2, %H2, [%1]\n" + : "+r" (cmp) // %0 + : "r" (target), // %1 + "r" (new) // %2 + : "memory"); + + /* if the old value is still the same unchanged, we won, otherwise we + * store the refreshed old value. + */ + ret = cmp.r[0] == bck.r[0] && cmp.r[1] == bck.r[1]; + if (unlikely(!ret)) { + /* update the old value on failure. Note that in this case the + * caller will likely relax and jump backwards so we don't care + * about this cost provided that it doesn't enlarge the fast + * code path. + */ + *(struct pair *)compare = cmp; + } + return ret; +} + +#elif defined(__SIZEOF_INT128__) && defined(_ARM_FEATURE_ATOMICS) // 128-bit and ARMv8.1-A will work + +/* According to https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html + * we can use atomics on __int128. The availability of CAS is defined there: + * https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html + * However these usually involve a function call which can be expensive for some + * cases, but gcc 10.2 and above can reroute the function call to either LL/SC for + * v8.0 or LSE for v8.1+, which allows to use a more scalable version on v8.1+ at + * the extra cost of a function call. + */ + +/* returns 0 on failure, non-zero on success */ +static __inline int __ha_cas_dw(void *target, void *compare, const void *set) +{ + return __atomic_compare_exchange_n((__int128*)target, (__int128*)compare, *(const __int128*)set, + 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +} + +#else // neither ARMv8.1-A atomics nor 128-bit atomics + +/* returns 0 on failure, non-zero on success */ +static __inline int __ha_cas_dw(void *target, void *compare, void *set) +{ + void *value[2]; + uint64_t tmp1, tmp2; + + __asm__ __volatile__("1:" + "ldxp %0, %1, [%4]\n" + "mov %2, %0\n" + "mov %3, %1\n" + "eor %0, %0, %5\n" + "eor %1, %1, %6\n" + "orr %1, %0, %1\n" + "mov %w0, #0\n" + "cbnz %1, 2f\n" + "stxp %w0, %7, %8, [%4]\n" + "cbnz %w0, 1b\n" + "mov %w0, #1\n" + "2:" + : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1]) + : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1]) + : "cc", "memory"); + + ((void **)(compare))[0] = value[0]; + ((void **)(compare))[1] = value[1]; + return (tmp1); +} +#endif // ARMv8.1-A atomics + +#else /* unknown / unhandled architecture, fall back to generic barriers */ + +#define __ha_barrier_atomic_load __sync_synchronize +#define __ha_barrier_atomic_store __sync_synchronize +#define __ha_barrier_atomic_full __sync_synchronize +#define __ha_barrier_load __sync_synchronize +#define __ha_barrier_store __sync_synchronize +#define __ha_barrier_full __sync_synchronize +/* Note: there is no generic DWCAS */ + +/* short-lived CPU relaxation */ +#define __ha_cpu_relax() ({ asm volatile(""); 1; }) + +#endif /* end of arch-specific barrier/dwcas */ + +static inline void __ha_compiler_barrier(void) +{ + __asm __volatile("" ::: "memory"); +} + +#endif /* USE_THREAD */ + + +/* fallbacks to remap all undefined _HA_ATOMIC_* on to their safe equivalent */ +#ifndef _HA_ATOMIC_BTR +#define _HA_ATOMIC_BTR HA_ATOMIC_BTR +#endif /* !_HA_ATOMIC_BTR */ + +#ifndef _HA_ATOMIC_BTS +#define _HA_ATOMIC_BTS HA_ATOMIC_BTS +#endif /* !_HA_ATOMIC_BTS */ + +#ifndef _HA_ATOMIC_CAS +#define _HA_ATOMIC_CAS HA_ATOMIC_CAS +#endif /* !_HA_ATOMIC_CAS */ + +#ifndef _HA_ATOMIC_DWCAS +#define _HA_ATOMIC_DWCAS HA_ATOMIC_DWCAS +#endif /* !_HA_ATOMIC_CAS */ + +#ifndef _HA_ATOMIC_ADD +#define _HA_ATOMIC_ADD HA_ATOMIC_ADD +#endif /* !_HA_ATOMIC_ADD */ + +#ifndef _HA_ATOMIC_ADD_FETCH +#define _HA_ATOMIC_ADD_FETCH HA_ATOMIC_ADD_FETCH +#endif /* !_HA_ATOMIC_ADD_FETCH */ + +#ifndef _HA_ATOMIC_FETCH_ADD +#define _HA_ATOMIC_FETCH_ADD HA_ATOMIC_FETCH_ADD +#endif /* !_HA_ATOMIC_FETCH_ADD */ + +#ifndef _HA_ATOMIC_SUB +#define _HA_ATOMIC_SUB HA_ATOMIC_SUB +#endif /* !_HA_ATOMIC_SUB */ + +#ifndef _HA_ATOMIC_SUB_FETCH +#define _HA_ATOMIC_SUB_FETCH HA_ATOMIC_SUB_FETCH +#endif /* !_HA_ATOMIC_SUB_FETCH */ + +#ifndef _HA_ATOMIC_FETCH_SUB +#define _HA_ATOMIC_FETCH_SUB HA_ATOMIC_FETCH_SUB +#endif /* !_HA_ATOMIC_FETCH_SUB */ + +#ifndef _HA_ATOMIC_INC +#define _HA_ATOMIC_INC HA_ATOMIC_INC +#endif /* !_HA_ATOMIC_INC */ + +#ifndef _HA_ATOMIC_DEC +#define _HA_ATOMIC_DEC HA_ATOMIC_DEC +#endif /* !_HA_ATOMIC_DEC */ + +#ifndef _HA_ATOMIC_AND +#define _HA_ATOMIC_AND HA_ATOMIC_AND +#endif /* !_HA_ATOMIC_AND */ + +#ifndef _HA_ATOMIC_AND_FETCH +#define _HA_ATOMIC_AND_FETCH HA_ATOMIC_AND_FETCH +#endif /* !_HA_ATOMIC_AND_FETCH */ + +#ifndef _HA_ATOMIC_FETCH_AND +#define _HA_ATOMIC_FETCH_AND HA_ATOMIC_FETCH_AND +#endif /* !_HA_ATOMIC_FETCH_AND */ + +#ifndef _HA_ATOMIC_OR +#define _HA_ATOMIC_OR HA_ATOMIC_OR +#endif /* !_HA_ATOMIC_OR */ + +#ifndef _HA_ATOMIC_OR_FETCH +#define _HA_ATOMIC_OR_FETCH HA_ATOMIC_OR_FETCH +#endif /* !_HA_ATOMIC_OR_FETCH */ + +#ifndef _HA_ATOMIC_FETCH_OR +#define _HA_ATOMIC_FETCH_OR HA_ATOMIC_FETCH_OR +#endif /* !_HA_ATOMIC_FETCH_OR */ + +#ifndef _HA_ATOMIC_XCHG +#define _HA_ATOMIC_XCHG HA_ATOMIC_XCHG +#endif /* !_HA_ATOMIC_XCHG */ + +#ifndef _HA_ATOMIC_STORE +#define _HA_ATOMIC_STORE HA_ATOMIC_STORE +#endif /* !_HA_ATOMIC_STORE */ + +#ifndef _HA_ATOMIC_LOAD +#define _HA_ATOMIC_LOAD HA_ATOMIC_LOAD +#endif /* !_HA_ATOMIC_LOAD */ + +#endif /* _HAPROXY_ATOMIC_H */ |