/* generic atomic operations used by progressive locks * * Copyright (C) 2012-2022 Willy Tarreau * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #ifndef PL_ATOMIC_OPS_H #define PL_ATOMIC_OPS_H /* The definitions below exist in two forms: * - fallback form (_pl_*) * - preferred form (pl_*) * * As a general rule, given that C11 atomics tend to offer more flexibility to * the compiler, these should set the preferred form, and the arch-specific * code should set the fallback code. But it's possible for arch-specific code * to set a preferred form, in which case it will simply be used over the other * ones. */ /* * Architecture-specific versions of the various operations */ /* * ###### ix86 / x86_64 below ###### */ #if defined(__i386__) || defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__) /* for compilers supporting condition flags on output, let's directly return them */ #if defined(__GCC_ASM_FLAG_OUTPUTS__) #define X86_COND_C_TO_REG(reg) "" #define X86_COND_Z_TO_REG(reg) "" #define X86_COND_NZ_TO_REG(reg) "" #define X86_COND_C_RESULT(var) "=@ccc"(var) #define X86_COND_Z_RESULT(var) "=@ccz"(var) #define X86_COND_NZ_RESULT(var) "=@ccnz"(var) #else #define X86_COND_C_TO_REG(reg) "sbb %" #reg ", %" #reg "\n\t" #define X86_COND_Z_TO_REG(reg) "sete %" #reg "\n\t" #define X86_COND_NZ_TO_REG(reg) "setne %" #reg "\n\t" #define X86_COND_C_RESULT(var) "=r"(var) #define X86_COND_Z_RESULT(var) "=qm"(var) #define X86_COND_NZ_RESULT(var) "=qm"(var) #endif /* CPU relaxation while waiting (PAUSE instruction on x86) */ #define pl_cpu_relax() do { \ asm volatile("rep;nop\n"); \ } while (0) /* full memory barrier using mfence when SSE2 is supported, falling back to * "lock add %esp" (gcc uses "lock add" or "lock or"). */ #if defined(__SSE2__) #define _pl_mb() do { \ asm volatile("mfence" ::: "memory"); \ } while (0) #elif defined(__x86_64__) #define _pl_mb() do { \ asm volatile("lock addl $0,0 (%%rsp)" ::: "memory", "cc"); \ } while (0) #else /* ix86 */ #define _pl_mb() do { \ asm volatile("lock addl $0,0 (%%esp)" ::: "memory", "cc"); \ } while (0) #endif /* end of pl_mb() case for sse2/x86_64/x86 */ /* load/store barriers are nops on x86 */ #define _pl_mb_load() do { asm volatile("" ::: "memory"); } while (0) #define _pl_mb_store() do { asm volatile("" ::: "memory"); } while (0) /* atomic full/load/store are also nops on x86 */ #define _pl_mb_ato() do { asm volatile("" ::: "memory"); } while (0) #define _pl_mb_ato_load() do { asm volatile("" ::: "memory"); } while (0) #define _pl_mb_ato_store() do { asm volatile("" ::: "memory"); } while (0) /* atomic load: on x86 it's just a volatile read */ #define _pl_load_lax(ptr) _pl_load(ptr) #define _pl_load(ptr) ({ typeof(*(ptr)) __ptr = *(volatile typeof(ptr))ptr; __ptr; }) /* atomic store: on x86 it's just a volatile write */ #define _pl_store_lax(ptr) _pl_store(ptr) #define _pl_store(ptr, x) do { *((volatile typeof(ptr))(ptr)) = (typeof(*ptr))(x); } while (0) /* increment integer value pointed to by pointer , and return non-zero if * result is non-null. */ #define _pl_inc_lax(ptr) _pl_inc(ptr) #define _pl_inc_acq(ptr) _pl_inc(ptr) #define _pl_inc_rel(ptr) _pl_inc(ptr) #define _pl_inc(ptr) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned char ret; \ asm volatile("lock incq %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned char ret; \ asm volatile("lock incl %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned char ret; \ asm volatile("lock incw %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret; \ asm volatile("lock incb %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_inc__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_inc__(__FILE__,__LINE__); \ 0; \ }) \ ) /* decrement integer value pointed to by pointer , and return non-zero if * result is non-null. */ #define _pl_dec_lax(ptr) _pl_dec(ptr) #define _pl_dec_acq(ptr) _pl_dec(ptr) #define _pl_dec_rel(ptr) _pl_dec(ptr) #define _pl_dec(ptr) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned char ret; \ asm volatile("lock decq %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned char ret; \ asm volatile("lock decl %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned char ret; \ asm volatile("lock decw %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret; \ asm volatile("lock decb %0\n" \ X86_COND_NZ_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_NZ_RESULT(ret) \ : \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_dec__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_dec__(__FILE__,__LINE__); \ 0; \ }) \ ) /* increment integer value pointed to by pointer , no return */ #define pl_inc_noret_lax(ptr) pl_inc_noret(ptr) #define pl_inc_noret_acq(ptr) pl_inc_noret(ptr) #define pl_inc_noret_rel(ptr) pl_inc_noret(ptr) #define pl_inc_noret(ptr) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock incq %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock incl %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock incw %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock incb %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_inc_noret__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_inc_noret__(__FILE__,__LINE__); \ } \ } while (0) /* decrement integer value pointed to by pointer , no return */ #define pl_dec_noret_lax(ptr) pl_dec_noret(ptr) #define pl_dec_noret_acq(ptr) pl_dec_noret(ptr) #define pl_dec_noret_rel(ptr) pl_dec_noret(ptr) #define pl_dec_noret(ptr) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock decq %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock decl %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock decw %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock decb %0\n" \ : "+m" (*(ptr)) \ : \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_dec_noret__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_dec_noret__(__FILE__,__LINE__); \ } \ } while (0) /* add integer constant to integer value pointed to by pointer , * no return. Size of is not checked. */ #define _pl_add_noret_lax(ptr, x) _pl_add_noret(ptr, x) #define _pl_add_noret_acq(ptr, x) _pl_add_noret(ptr, x) #define _pl_add_noret_rel(ptr, x) _pl_add_noret(ptr, x) #define _pl_add_noret(ptr, x) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock addq %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned long)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock addl %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned int)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock addw %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned short)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock addb %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned char)(x)) \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_add__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_add__(__FILE__,__LINE__); \ } \ } while (0) /* subtract integer constant from integer value pointed to by pointer * , no return. Size of is not checked. */ #define _pl_sub_noret_lax(ptr, x) _pl_sub_noret(ptr, x) #define _pl_sub_noret_acq(ptr, x) _pl_sub_noret(ptr, x) #define _pl_sub_noret_rel(ptr, x) _pl_sub_noret(ptr, x) #define _pl_sub_noret(ptr, x) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock subq %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned long)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock subl %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned int)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock subw %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned short)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock subb %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned char)(x)) \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_sub__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_sub__(__FILE__,__LINE__); \ } \ } while (0) /* binary and integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #define _pl_and_noret_lax(ptr, x) _pl_and_noret(ptr, x) #define _pl_and_noret_acq(ptr, x) _pl_and_noret(ptr, x) #define _pl_and_noret_rel(ptr, x) _pl_and_noret(ptr, x) #define _pl_and_noret(ptr, x) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock andq %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned long)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock andl %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned int)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock andw %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned short)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock andb %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned char)(x)) \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_and__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_and__(__FILE__,__LINE__); \ } \ } while (0) /* binary or integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #define _pl_or_noret_lax(ptr, x) _pl_or_noret(ptr, x) #define _pl_or_noret_acq(ptr, x) _pl_or_noret(ptr, x) #define _pl_or_noret_rel(ptr, x) _pl_or_noret(ptr, x) #define _pl_or_noret(ptr, x) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock orq %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned long)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock orl %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned int)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock orw %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned short)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock orb %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned char)(x)) \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_or__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_or__(__FILE__,__LINE__); \ } \ } while (0) /* binary xor integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #define _pl_xor_noret_lax(ptr, x) _pl_xor_noret(ptr, x) #define _pl_xor_noret_acq(ptr, x) _pl_xor_noret(ptr, x) #define _pl_xor_noret_rel(ptr, x) _pl_xor_noret(ptr, x) #define _pl_xor_noret(ptr, x) do { \ if (sizeof(long) == 8 && sizeof(*(ptr)) == 8) { \ asm volatile("lock xorq %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned long)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 4) { \ asm volatile("lock xorl %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned int)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 2) { \ asm volatile("lock xorw %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned short)(x)) \ : "cc"); \ } else if (sizeof(*(ptr)) == 1) { \ asm volatile("lock xorb %1, %0\n" \ : "+m" (*(ptr)) \ : "er" ((unsigned char)(x)) \ : "cc"); \ } else { \ void __unsupported_argument_size_for_pl_xor__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_xor__(__FILE__,__LINE__); \ } \ } while (0) /* test and reset bit in integer value pointed to by pointer . Returns * 0 if the bit was not set, or ~0 of the same type as *ptr if it was set. Note * that there is no 8-bit equivalent operation. */ #define pl_btr_lax(ptr, bit) pl_btr(ptr, bit) #define pl_btr_acq(ptr, bit) pl_btr(ptr, bit) #define pl_btr_rel(ptr, bit) pl_btr(ptr, bit) #define pl_btr(ptr, bit) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret; \ asm volatile("lock btrq %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned long)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret; \ asm volatile("lock btrl %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned int)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret; \ asm volatile("lock btrw %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned short)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_btr__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_btr__(__FILE__,__LINE__); \ 0; \ }) \ ) /* test and set bit in integer value pointed to by pointer . Returns * 0 if the bit was not set, or ~0 of the same type as *ptr if it was set. Note * that there is no 8-bit equivalent operation. */ #define pl_bts_lax(ptr, bit) pl_bts(ptr, bit) #define pl_bts_acq(ptr, bit) pl_bts(ptr, bit) #define pl_bts_rel(ptr, bit) pl_bts(ptr, bit) #define pl_bts(ptr, bit) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret; \ asm volatile("lock btsq %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned long)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret; \ asm volatile("lock btsl %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned int)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret; \ asm volatile("lock btsw %2, %0\n\t" \ X86_COND_C_TO_REG(1) \ : "+m" (*(ptr)), X86_COND_C_RESULT(ret) \ : "Ir" ((unsigned short)(bit)) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_bts__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_bts__(__FILE__,__LINE__); \ 0; \ }) \ ) /* Note: for an unclear reason, gcc's __sync_fetch_and_add() implementation * produces less optimal than hand-crafted asm code so let's implement here the * operations we need for the most common archs. */ /* fetch-and-add: fetch integer value pointed to by pointer , add to * to <*ptr> and return the previous value. * => THIS IS LEGACY, USE _pl_ldadd() INSTEAD. */ #define _pl_xadd(ptr, x) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret = (unsigned long)(x); \ asm volatile("lock xaddq %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret = (unsigned int)(x); \ asm volatile("lock xaddl %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret = (unsigned short)(x); \ asm volatile("lock xaddw %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret = (unsigned char)(x); \ asm volatile("lock xaddb %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_xadd__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_xadd__(__FILE__,__LINE__); \ 0; \ }) \ ) /* fetch-and-add: fetch integer value pointed to by pointer , add to * to <*ptr> and return the previous value. */ #define _pl_ldadd_lax(ptr, x) _pl_ldadd(ptr, x) #define _pl_ldadd_acq(ptr, x) _pl_ldadd(ptr, x) #define _pl_ldadd_rel(ptr, x) _pl_ldadd(ptr, x) #define _pl_ldadd(ptr, x) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret = (unsigned long)(x); \ asm volatile("lock xaddq %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret = (unsigned int)(x); \ asm volatile("lock xaddl %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret = (unsigned short)(x); \ asm volatile("lock xaddw %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret = (unsigned char)(x); \ asm volatile("lock xaddb %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_ldadd__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_ldadd__(__FILE__,__LINE__); \ 0; \ }) \ ) /* fetch-and-sub: fetch integer value pointed to by pointer , add - to * to <*ptr> and return the previous value. */ #define _pl_ldsub_lax(ptr, x) _pl_ldsub(ptr, x) #define _pl_ldsub_acq(ptr, x) _pl_ldsub(ptr, x) #define _pl_ldsub_rel(ptr, x) _pl_ldsub(ptr, x) #define _pl_ldsub(ptr, x) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret = (unsigned long)(-x); \ asm volatile("lock xaddq %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret = (unsigned int)(-x); \ asm volatile("lock xaddl %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret = (unsigned short)(-x); \ asm volatile("lock xaddw %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret = (unsigned char)(-x); \ asm volatile("lock xaddb %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_ldsub__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_ldsub__(__FILE__,__LINE__); \ 0; \ }) \ ) /* exchange value with integer value pointed to by pointer , and return * previous <*ptr> value. must be of the same size as <*ptr>. */ #define _pl_xchg(ptr, x) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret = (unsigned long)(x); \ asm volatile("xchgq %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret = (unsigned int)(x); \ asm volatile("xchgl %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret = (unsigned short)(x); \ asm volatile("xchgw %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret = (unsigned char)(x); \ asm volatile("xchgb %0, %1\n" \ : "=r" (ret), "+m" (*(ptr)) \ : "0" (ret) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_xchg__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_xchg__(__FILE__,__LINE__); \ 0; \ }) \ ) /* compare integer value <*ptr> with and exchange it with if * it matches, and return . and must be of the same size as * <*ptr>. */ #define _pl_cmpxchg(ptr, old, new) ( \ (sizeof(long) == 8 && sizeof(*(ptr)) == 8) ? ({ \ unsigned long ret; \ asm volatile("lock cmpxchgq %2,%1" \ : "=a" (ret), "+m" (*(ptr)) \ : "r" ((unsigned long)(new)), \ "0" ((unsigned long)(old)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 4) ? ({ \ unsigned int ret; \ asm volatile("lock cmpxchgl %2,%1" \ : "=a" (ret), "+m" (*(ptr)) \ : "r" ((unsigned int)(new)), \ "0" ((unsigned int)(old)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 2) ? ({ \ unsigned short ret; \ asm volatile("lock cmpxchgw %2,%1" \ : "=a" (ret), "+m" (*(ptr)) \ : "r" ((unsigned short)(new)), \ "0" ((unsigned short)(old)) \ : "cc"); \ ret; /* return value */ \ }) : (sizeof(*(ptr)) == 1) ? ({ \ unsigned char ret; \ asm volatile("lock cmpxchgb %2,%1" \ : "=a" (ret), "+m" (*(ptr)) \ : "r" ((unsigned char)(new)), \ "0" ((unsigned char)(old)) \ : "cc"); \ ret; /* return value */ \ }) : ({ \ void __unsupported_argument_size_for_pl_cmpxchg__(char *,int); \ if (sizeof(*(ptr)) != 1 && sizeof(*(ptr)) != 2 && \ sizeof(*(ptr)) != 4 && (sizeof(long) != 8 || sizeof(*(ptr)) != 8)) \ __unsupported_argument_size_for_pl_cmpxchg__(__FILE__,__LINE__); \ 0; \ }) \ ) /* * ##### ARM64 (aarch64) below ##### */ #elif defined(__aarch64__) /* This was shown to improve fairness on modern ARMv8 such as Neoverse N1 */ #define pl_cpu_relax() do { \ asm volatile("isb" ::: "memory"); \ } while (0) /* full/load/store barriers */ #define _pl_mb() do { asm volatile("dmb ish" ::: "memory"); } while (0) #define _pl_mb_load() do { asm volatile("dmb ishld" ::: "memory"); } while (0) #define _pl_mb_store() do { asm volatile("dmb ishst" ::: "memory"); } while (0) /* atomic full/load/store */ #define _pl_mb_ato() do { asm volatile("dmb ish" ::: "memory"); } while (0) #define _pl_mb_ato_load() do { asm volatile("dmb ishld" ::: "memory"); } while (0) #define _pl_mb_ato_store() do { asm volatile("dmb ishst" ::: "memory"); } while (0) #endif // end of arch-specific code /* * Generic code using the C11 __atomic API for functions not defined above. * These are usable from gcc-4.7 and clang. We'll simply rely on the macros * defining the memory orders to detect them. All operations are not * necessarily defined, so some fallbacks to the default methods might still * be necessary. */ #if defined(__ATOMIC_RELAXED) && defined(__ATOMIC_CONSUME) && defined(__ATOMIC_ACQUIRE) && \ defined(__ATOMIC_RELEASE) && defined(__ATOMIC_ACQ_REL) && defined(__ATOMIC_SEQ_CST) /* compiler-only memory barrier, for use around locks */ #ifndef pl_barrier #define pl_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST) #endif /* full memory barrier */ #ifndef pl_mb #define pl_mb() __atomic_thread_fence(__ATOMIC_SEQ_CST) #endif /* atomic load */ #ifndef pl_load_lax #define pl_load_lax(ptr) __atomic_load_n(ptr, __ATOMIC_RELAXED) #endif #ifndef pl_load #define pl_load(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE) #endif /* atomic store */ #ifndef pl_store_lax #define pl_store_lax(ptr, x) __atomic_store_n((ptr), (x), __ATOMIC_RELAXED) #endif #ifndef pl_store #define pl_store(ptr, x) __atomic_store_n((ptr), (x), __ATOMIC_RELEASE) #endif /* increment integer value pointed to by pointer , and return non-zero if * result is non-null. */ #ifndef pl_inc_lax #define pl_inc_lax(ptr) (__atomic_add_fetch((ptr), 1, __ATOMIC_RELAXED) != 0) #endif #ifndef pl_inc_acq #define pl_inc_acq(ptr) (__atomic_add_fetch((ptr), 1, __ATOMIC_ACQUIRE) != 0) #endif #ifndef pl_inc_rel #define pl_inc_rel(ptr) (__atomic_add_fetch((ptr), 1, __ATOMIC_RELEASE) != 0) #endif #ifndef pl_inc #define pl_inc(ptr) (__atomic_add_fetch((ptr), 1, __ATOMIC_SEQ_CST) != 0) #endif /* decrement integer value pointed to by pointer , and return non-zero if * result is non-null. */ #ifndef pl_dec_lax #define pl_dec_lax(ptr) (__atomic_sub_fetch((ptr), 1, __ATOMIC_RELAXED) != 0) #endif #ifndef pl_dec_acq #define pl_dec_acq(ptr) (__atomic_sub_fetch((ptr), 1, __ATOMIC_ACQUIRE) != 0) #endif #ifndef pl_dec_rel #define pl_dec_rel(ptr) (__atomic_sub_fetch((ptr), 1, __ATOMIC_RELEASE) != 0) #endif #ifndef pl_dec #define pl_dec(ptr) (__atomic_sub_fetch((ptr), 1, __ATOMIC_SEQ_CST) != 0) #endif /* increment integer value pointed to by pointer , no return */ #ifndef pl_inc_noret_lax #define pl_inc_noret_lax(ptr) ((void)__atomic_add_fetch((ptr), 1, __ATOMIC_RELAXED)) #endif #ifndef pl_inc_noret_acq #define pl_inc_noret_acq(ptr) ((void)__atomic_add_fetch((ptr), 1, __ATOMIC_ACQUIRE)) #endif #ifndef pl_inc_noret_rel #define pl_inc_noret_relc(ptr) ((void)__atomic_add_fetch((ptr), 1, __ATOMIC_RELEASE)) #endif #ifndef pl_inc_noret #define pl_inc_noret(ptr) ((void)__atomic_add_fetch((ptr), 1, __ATOMIC_SEQ_CST)) #endif /* decrement integer value pointed to by pointer , no return */ #ifndef pl_dec_noret_lax #define pl_dec_noret_lax(ptr) ((void)__atomic_sub_fetch((ptr), 1, __ATOMIC_RELAXED)) #endif #ifndef pl_dec_noret_acq #define pl_dec_noret_acq(ptr) ((void)__atomic_sub_fetch((ptr), 1, __ATOMIC_ACQUIRE)) #endif #ifndef pl_dec_noret_rel #define pl_dec_noret_relc(ptr) ((void)__atomic_sub_fetch((ptr), 1, __ATOMIC_RELEASE)) #endif #ifndef pl_dec_noret #define pl_dec_noret(ptr) ((void)__atomic_sub_fetch((ptr), 1, __ATOMIC_SEQ_CST)) #endif /* add integer constant to integer value pointed to by pointer , * no return. Size of is not checked. */ #ifndef pl_add_lax #define pl_add_lax(ptr, x) (__atomic_add_fetch((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_add_acq #define pl_add_acq(ptr, x) (__atomic_add_fetch((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_add_rel #define pl_add_relc(ptr, x) (__atomic_add_fetch((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_add #define pl_add(ptr, x) (__atomic_add_fetch((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* subtract integer constant from integer value pointed to by pointer * , no return. Size of is not checked. */ #ifndef pl_sub_lax #define pl_sub_lax(ptr, x) (__atomic_sub_fetch((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_sub_acq #define pl_sub_acq(ptr, x) (__atomic_sub_fetch((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_sub_rel #define pl_sub_relc(ptr, x) (__atomic_sub_fetch((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_sub #define pl_sub(ptr, x) (__atomic_sub_fetch((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* binary and integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #ifndef pl_and_lax #define pl_and_lax(ptr, x) (__atomic_and_fetch((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_and_acq #define pl_and_acq(ptr, x) (__atomic_and_fetch((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_and_rel #define pl_and_relc(ptr, x) (__atomic_and_fetch((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_and #define pl_and(ptr, x) (__atomic_and_fetch((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* binary or integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #ifndef pl_or_lax #define pl_or_lax(ptr, x) (__atomic_or_fetch((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_or_acq #define pl_or_acq(ptr, x) (__atomic_or_fetch((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_or_rel #define pl_or_relc(ptr, x) (__atomic_or_fetch((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_or #define pl_or(ptr, x) (__atomic_or_fetch((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* binary xor integer value pointed to by pointer with constant , no * return. Size of is not checked. */ #ifndef pl_xor_lax #define pl_xor_lax(ptr, x) (__atomic_xor_fetch((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_xor_acq #define pl_xor_acq(ptr, x) (__atomic_xor_fetch((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_xor_rel #define pl_xor_relc(ptr, x) (__atomic_xor_fetch((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_xor #define pl_xor(ptr, x) (__atomic_xor_fetch((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* fetch-and-add: fetch integer value pointed to by pointer , add to * to <*ptr> and return the previous value. * => THIS IS LEGACY, USE pl_ldadd() INSTEAD. */ #ifndef pl_xadd #define pl_xadd(ptr, x) (__atomic_fetch_add((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* exchange value with integer value pointed to by pointer , and return * previous <*ptr> value. must be of the same size as <*ptr>. */ #ifndef pl_xchg #define pl_xchg(ptr, x) (__atomic_exchange_n((ptr), (x), __ATOMIC_SEQ_CST)) #endif /* compare integer value <*ptr> with and exchange it with if * it matches, and return . and must be of the same size as * <*ptr>. */ #ifndef pl_cmpxchg #define pl_cmpxchg(ptr, old, new) ({ \ typeof(*ptr) __old = (old); \ __atomic_compare_exchange_n((ptr), &__old, (new), 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); \ __old; }) #endif /* fetch-and-add: fetch integer value pointed to by pointer , add to * to <*ptr> and return the previous value. */ #ifndef pl_ldadd_lax #define pl_ldadd_lax(ptr, x) (__atomic_fetch_add((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_ldadd_acq #define pl_ldadd_acq(ptr, x) (__atomic_fetch_add((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_ldadd_rel #define pl_ldadd_relc(ptr, x) (__atomic_fetch_add((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_ldadd #define pl_ldadd(ptr, x) (__atomic_fetch_add((ptr), (x), __ATOMIC_SEQ_CST)) #endif #ifndef pl_ldand_lax #define pl_ldand_lax(ptr, x) (__atomic_fetch_and((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_ldand_acq #define pl_ldand_acq(ptr, x) (__atomic_fetch_and((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_ldand_rel #define pl_ldand_relc(ptr, x) (__atomic_fetch_and((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_ldand #define pl_ldand(ptr, x) (__atomic_fetch_and((ptr), (x), __ATOMIC_SEQ_CST)) #endif #ifndef pl_ldor_lax #define pl_ldor_lax(ptr, x) (__atomic_fetch_or((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_ldor_acq #define pl_ldor_acq(ptr, x) (__atomic_fetch_or((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_ldor_rel #define pl_ldor_relc(ptr, x) (__atomic_fetch_or((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_ldor #define pl_ldor(ptr, x) (__atomic_fetch_or((ptr), (x), __ATOMIC_SEQ_CST)) #endif #ifndef pl_ldsub_lax #define pl_ldsub_lax(ptr, x) (__atomic_fetch_sub((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_ldsub_acq #define pl_ldsub_acq(ptr, x) (__atomic_fetch_sub((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_ldsub_rel #define pl_ldsub_relc(ptr, x) (__atomic_fetch_sub((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_ldsub #define pl_ldsub(ptr, x) (__atomic_fetch_sub((ptr), (x), __ATOMIC_SEQ_CST)) #endif #ifndef pl_ldxor_lax #define pl_ldxor_lax(ptr, x) (__atomic_fetch_xor((ptr), (x), __ATOMIC_RELAXED)) #endif #ifndef pl_ldxor_acq #define pl_ldxor_acq(ptr, x) (__atomic_fetch_xor((ptr), (x), __ATOMIC_ACQUIRE)) #endif #ifndef pl_ldxor_rel #define pl_ldxor_relc(ptr, x) (__atomic_fetch_xor((ptr), (x), __ATOMIC_RELEASE)) #endif #ifndef pl_ldxor #define pl_ldxor(ptr, x) (__atomic_fetch_xor((ptr), (x), __ATOMIC_SEQ_CST)) #endif #endif /* end of C11 atomics */ /* * Automatically remap to fallback code when available. This allows the arch * specific code above to be used as an immediate fallback for missing C11 * definitions. Everything not defined will use the generic code at the end. */ #if !defined(pl_cpu_relax) && defined(_pl_cpu_relax) # define pl_cpu_relax _pl_cpu_relax #endif #if !defined(pl_barrier) && defined(_pl_barrier) # define pl_barrier _pl_barrier #endif #if !defined(pl_mb) && defined(_pl_mb) # define pl_mb _pl_mb #endif #if !defined(pl_mb_load) && defined(_pl_mb_load) # define pl_mb_load _pl_mb_load #endif #if !defined(pl_mb_store) && defined(_pl_mb_store) # define pl_mb_store _pl_mb_store #endif #if !defined(pl_mb_ato) && defined(_pl_mb_ato) # define pl_mb_ato _pl_mb_ato #endif #if !defined(pl_mb_ato_load) && defined(_pl_mb_ato_load) # define pl_mb_ato_load _pl_mb_ato_load #endif #if !defined(pl_mb_ato_store) && defined(_pl_mb_ato_store) # define pl_mb_ato_store _pl_mb_ato_store #endif #if !defined(pl_load) && defined(_pl_load) #define pl_load _pl_load #endif #if !defined(pl_load_lax) && defined(_pl_load_lax) #define pl_load_lax _pl_load_lax #endif #if !defined(pl_store) && defined(_pl_store) #define pl_store _pl_store #endif #if !defined(pl_store_lax) && defined(_pl_store_lax) #define pl_store_lax _pl_store_lax #endif #if !defined(pl_inc_noret_lax) && defined(_pl_inc_noret_lax) # define pl_inc_noret_lax _pl_inc_noret_lax #endif #if !defined(pl_inc_noret_acq) && defined(_pl_inc_noret_acq) # define pl_inc_noret_acq _pl_inc_noret_acq #endif #if !defined(pl_inc_noret_rel) && defined(_pl_inc_noret_rel) # define pl_inc_noret_rel _pl_inc_noret_rel #endif #if !defined(pl_inc_noret) && defined(_pl_inc_noret) # define pl_inc_noret _pl_inc_noret #endif #if !defined(pl_dec_noret_lax) && defined(_pl_dec_noret_lax) # define pl_dec_noret_lax _pl_dec_noret_lax #endif #if !defined(pl_dec_noret_acq) && defined(_pl_dec_noret_acq) # define pl_dec_noret_acq _pl_dec_noret_acq #endif #if !defined(pl_dec_noret_rel) && defined(_pl_dec_noret_rel) # define pl_dec_noret_rel _pl_dec_noret_rel #endif #if !defined(pl_dec_noret) && defined(_pl_dec_noret) # define pl_dec_noret _pl_dec_noret #endif #if !defined(pl_inc_lax) && defined(_pl_inc_lax) # define pl_inc_lax _pl_inc_lax #endif #if !defined(pl_inc_acq) && defined(_pl_inc_acq) # define pl_inc_acq _pl_inc_acq #endif #if !defined(pl_inc_rel) && defined(_pl_inc_rel) # define pl_inc_rel _pl_inc_rel #endif #if !defined(pl_inc) && defined(_pl_inc) # define pl_inc _pl_inc #endif #if !defined(pl_dec_lax) && defined(_pl_dec_lax) # define pl_dec_lax _pl_dec_lax #endif #if !defined(pl_dec_acq) && defined(_pl_dec_acq) # define pl_dec_acq _pl_dec_acq #endif #if !defined(pl_dec_rel) && defined(_pl_dec_rel) # define pl_dec_rel _pl_dec_rel #endif #if !defined(pl_dec) && defined(_pl_dec) # define pl_dec _pl_dec #endif #if !defined(pl_add_lax) && defined(_pl_add_lax) # define pl_add_lax _pl_add_lax #endif #if !defined(pl_add_acq) && defined(_pl_add_acq) # define pl_add_acq _pl_add_acq #endif #if !defined(pl_add_rel) && defined(_pl_add_rel) # define pl_add_rel _pl_add_rel #endif #if !defined(pl_add) && defined(_pl_add) # define pl_add _pl_add #endif #if !defined(pl_add_noret_lax) && defined(_pl_add_noret_lax) # define pl_add_noret_lax _pl_add_noret_lax #endif #if !defined(pl_add_noret_acq) && defined(_pl_add_noret_acq) # define pl_add_noret_acq _pl_add_noret_acq #endif #if !defined(pl_add_noret_rel) && defined(_pl_add_noret_rel) # define pl_add_noret_rel _pl_add_noret_rel #endif #if !defined(pl_add_noret) && defined(_pl_add_noret) # define pl_add_noret _pl_add_noret #endif #if !defined(pl_and_lax) && defined(_pl_and_lax) # define pl_and_lax _pl_and_lax #endif #if !defined(pl_and_acq) && defined(_pl_and_acq) # define pl_and_acq _pl_and_acq #endif #if !defined(pl_and_rel) && defined(_pl_and_rel) # define pl_and_rel _pl_and_rel #endif #if !defined(pl_and) && defined(_pl_and) # define pl_and _pl_and #endif #if !defined(pl_and_noret_lax) && defined(_pl_and_noret_lax) # define pl_and_noret_lax _pl_and_noret_lax #endif #if !defined(pl_and_noret_acq) && defined(_pl_and_noret_acq) # define pl_and_noret_acq _pl_and_noret_acq #endif #if !defined(pl_and_noret_rel) && defined(_pl_and_noret_rel) # define pl_and_noret_rel _pl_and_noret_rel #endif #if !defined(pl_and_noret) && defined(_pl_and_noret) # define pl_and_noret _pl_and_noret #endif #if !defined(pl_or_lax) && defined(_pl_or_lax) # define pl_or_lax _pl_or_lax #endif #if !defined(pl_or_acq) && defined(_pl_or_acq) # define pl_or_acq _pl_or_acq #endif #if !defined(pl_or_rel) && defined(_pl_or_rel) # define pl_or_rel _pl_or_rel #endif #if !defined(pl_or) && defined(_pl_or) # define pl_or _pl_or #endif #if !defined(pl_or_noret_lax) && defined(_pl_or_noret_lax) # define pl_or_noret_lax _pl_or_noret_lax #endif #if !defined(pl_or_noret_acq) && defined(_pl_or_noret_acq) # define pl_or_noret_acq _pl_or_noret_acq #endif #if !defined(pl_or_noret_rel) && defined(_pl_or_noret_rel) # define pl_or_noret_rel _pl_or_noret_rel #endif #if !defined(pl_or_noret) && defined(_pl_or_noret) # define pl_or_noret _pl_or_noret #endif #if !defined(pl_xor_lax) && defined(_pl_xor_lax) # define pl_xor_lax _pl_xor_lax #endif #if !defined(pl_xor_acq) && defined(_pl_xor_acq) # define pl_xor_acq _pl_xor_acq #endif #if !defined(pl_xor_rel) && defined(_pl_xor_rel) # define pl_xor_rel _pl_xor_rel #endif #if !defined(pl_xor) && defined(_pl_xor) # define pl_xor _pl_xor #endif #if !defined(pl_xor_noret_lax) && defined(_pl_xor_noret_lax) # define pl_xor_noret_lax _pl_xor_noret_lax #endif #if !defined(pl_xor_noret_acq) && defined(_pl_xor_noret_acq) # define pl_xor_noret_acq _pl_xor_noret_acq #endif #if !defined(pl_xor_noret_rel) && defined(_pl_xor_noret_rel) # define pl_xor_noret_rel _pl_xor_noret_rel #endif #if !defined(pl_xor_noret) && defined(_pl_xor_noret) # define pl_xor_noret _pl_xor_noret #endif #if !defined(pl_sub_lax) && defined(_pl_sub_lax) # define pl_sub_lax _pl_sub_lax #endif #if !defined(pl_sub_acq) && defined(_pl_sub_acq) # define pl_sub_acq _pl_sub_acq #endif #if !defined(pl_sub_rel) && defined(_pl_sub_rel) # define pl_sub_rel _pl_sub_rel #endif #if !defined(pl_sub) && defined(_pl_sub) # define pl_sub _pl_sub #endif #if !defined(pl_sub_noret_lax) && defined(_pl_sub_noret_lax) # define pl_sub_noret_lax _pl_sub_noret_lax #endif #if !defined(pl_sub_noret_acq) && defined(_pl_sub_noret_acq) # define pl_sub_noret_acq _pl_sub_noret_acq #endif #if !defined(pl_sub_noret_rel) && defined(_pl_sub_noret_rel) # define pl_sub_noret_rel _pl_sub_noret_rel #endif #if !defined(pl_sub_noret) && defined(_pl_sub_noret) # define pl_sub_noret _pl_sub_noret #endif #if !defined(pl_btr_lax) && defined(_pl_btr_lax) # define pl_btr_lax _pl_btr_lax #endif #if !defined(pl_btr_acq) && defined(_pl_btr_acq) # define pl_btr_acq _pl_btr_acq #endif #if !defined(pl_btr_rel) && defined(_pl_btr_rel) # define pl_btr_rel _pl_btr_rel #endif #if !defined(pl_btr) && defined(_pl_btr) # define pl_btr _pl_btr #endif #if !defined(pl_bts_lax) && defined(_pl_bts_lax) # define pl_bts_lax _pl_bts_lax #endif #if !defined(pl_bts_acq) && defined(_pl_bts_acq) # define pl_bts_acq _pl_bts_acq #endif #if !defined(pl_bts_rel) && defined(_pl_bts_rel) # define pl_bts_rel _pl_bts_rel #endif #if !defined(pl_bts) && defined(_pl_bts) # define pl_bts _pl_bts #endif #if !defined(pl_xadd) && defined(_pl_xadd) # define pl_xadd _pl_xadd #endif #if !defined(pl_cmpxchg) && defined(_pl_cmpxchg) # define pl_cmpxchg _pl_cmpxchg #endif #if !defined(pl_xchg) && defined(_pl_xchg) # define pl_xchg _pl_xchg #endif #if !defined(pl_ldadd_lax) && defined(_pl_ldadd_lax) # define pl_ldadd_lax _pl_ldadd_lax #endif #if !defined(pl_ldadd_acq) && defined(_pl_ldadd_acq) # define pl_ldadd_acq _pl_ldadd_acq #endif #if !defined(pl_ldadd_rel) && defined(_pl_ldadd_rel) # define pl_ldadd_rel _pl_ldadd_rel #endif #if !defined(pl_ldadd) && defined(_pl_ldadd) # define pl_ldadd _pl_ldadd #endif #if !defined(pl_ldand_lax) && defined(_pl_ldand_lax) # define pl_ldand_lax _pl_ldand_lax #endif #if !defined(pl_ldand_acq) && defined(_pl_ldand_acq) # define pl_ldand_acq _pl_ldand_acq #endif #if !defined(pl_ldand_rel) && defined(_pl_ldand_rel) # define pl_ldand_rel _pl_ldand_rel #endif #if !defined(pl_ldand) && defined(_pl_ldand) # define pl_ldand _pl_ldand #endif #if !defined(pl_ldor_lax) && defined(_pl_ldor_lax) # define pl_ldor_lax _pl_ldor_lax #endif #if !defined(pl_ldor_acq) && defined(_pl_ldor_acq) # define pl_ldor_acq _pl_ldor_acq #endif #if !defined(pl_ldor_rel) && defined(_pl_ldor_rel) # define pl_ldor_rel _pl_ldor_rel #endif #if !defined(pl_ldor) && defined(_pl_ldor) # define pl_ldor _pl_ldor #endif #if !defined(pl_ldxor_lax) && defined(_pl_ldxor_lax) # define pl_ldxor_lax _pl_ldxor_lax #endif #if !defined(pl_ldxor_acq) && defined(_pl_ldxor_acq) # define pl_ldxor_acq _pl_ldxor_acq #endif #if !defined(pl_ldxor_rel) && defined(_pl_ldxor_rel) # define pl_ldxor_rel _pl_ldxor_rel #endif #if !defined(pl_ldxor) && defined(_pl_ldxor) # define pl_ldxor _pl_ldxor #endif #if !defined(pl_ldsub_lax) && defined(_pl_ldsub_lax) # define pl_ldsub_lax _pl_ldsub_lax #endif #if !defined(pl_ldsub_acq) && defined(_pl_ldsub_acq) # define pl_ldsub_acq _pl_ldsub_acq #endif #if !defined(pl_ldsub_rel) && defined(_pl_ldsub_rel) # define pl_ldsub_rel _pl_ldsub_rel #endif #if !defined(pl_ldsub) && defined(_pl_ldsub) # define pl_ldsub _pl_ldsub #endif /* * Generic code using the __sync API for everything not defined above. */ /* CPU relaxation while waiting */ #ifndef pl_cpu_relax #define pl_cpu_relax() do { \ asm volatile(""); \ } while (0) #endif /* compiler-only memory barrier, for use around locks */ #ifndef pl_barrier #define pl_barrier() do { \ asm volatile("" ::: "memory"); \ } while (0) #endif /* full memory barrier */ #ifndef pl_mb #define pl_mb() do { \ __sync_synchronize(); \ } while (0) #endif #ifndef pl_mb_load #define pl_mb_load() pl_mb() #endif #ifndef pl_mb_store #define pl_mb_store() pl_mb() #endif #ifndef pl_mb_ato #define pl_mb_ato() pl_mb() #endif #ifndef pl_mb_ato_load #define pl_mb_ato_load() pl_mb_ato() #endif #ifndef pl_mb_ato_store #define pl_mb_ato_store() pl_mb_ato() #endif /* atomic load: volatile after a load barrier */ #ifndef pl_load #define pl_load(ptr) ({ \ typeof(*(ptr)) __pl_ret = ({ \ pl_mb_load(); \ *(volatile typeof(ptr))ptr; \ }); \ __pl_ret; \ }) #endif /* atomic store, old style using a CAS */ #ifndef pl_store #define pl_store(ptr, x) do { \ typeof((ptr)) __pl_ptr = (ptr); \ typeof((x)) __pl_x = (x); \ typeof(*(ptr)) __pl_old; \ do { \ __pl_old = *__pl_ptr; \ } while (!__sync_bool_compare_and_swap(__pl_ptr, __pl_old, __pl_x)); \ } while (0) #endif #ifndef pl_inc_noret #define pl_inc_noret(ptr) do { __sync_add_and_fetch((ptr), 1); } while (0) #endif #ifndef pl_dec_noret #define pl_dec_noret(ptr) do { __sync_sub_and_fetch((ptr), 1); } while (0) #endif #ifndef pl_inc #define pl_inc(ptr) ({ __sync_add_and_fetch((ptr), 1); }) #endif #ifndef pl_dec #define pl_dec(ptr) ({ __sync_sub_and_fetch((ptr), 1); }) #endif #ifndef pl_add #define pl_add(ptr, x) ({ __sync_add_and_fetch((ptr), (x)); }) #endif #ifndef pl_and #define pl_and(ptr, x) ({ __sync_and_and_fetch((ptr), (x)); }) #endif #ifndef pl_or #define pl_or(ptr, x) ({ __sync_or_and_fetch((ptr), (x)); }) #endif #ifndef pl_xor #define pl_xor(ptr, x) ({ __sync_xor_and_fetch((ptr), (x)); }) #endif #ifndef pl_sub #define pl_sub(ptr, x) ({ __sync_sub_and_fetch((ptr), (x)); }) #endif #ifndef pl_btr #define pl_btr(ptr, bit) ({ typeof(*(ptr)) __pl_t = ((typeof(*(ptr)))1) << (bit); \ __sync_fetch_and_and((ptr), ~__pl_t) & __pl_t; \ }) #endif #ifndef pl_bts #define pl_bts(ptr, bit) ({ typeof(*(ptr)) __pl_t = ((typeof(*(ptr)))1) << (bit); \ __sync_fetch_and_or((ptr), __pl_t) & __pl_t; \ }) #endif #ifndef pl_xadd #define pl_xadd(ptr, x) ({ __sync_fetch_and_add((ptr), (x)); }) #endif #ifndef pl_cmpxchg #define pl_cmpxchg(ptr, o, n) ({ __sync_val_compare_and_swap((ptr), (o), (n)); }) #endif #ifndef pl_xchg #define pl_xchg(ptr, x) ({ \ typeof((ptr)) __pl_ptr = (ptr); \ typeof((x)) __pl_x = (x); \ typeof(*(ptr)) __pl_old; \ do { \ __pl_old = *__pl_ptr; \ } while (!__sync_bool_compare_and_swap(__pl_ptr, __pl_old, __pl_x)); \ __pl_old; \ }) #endif #ifndef pl_ldadd #define pl_ldadd(ptr, x) ({ __sync_fetch_and_add((ptr), (x)); }) #endif #ifndef pl_ldand #define pl_ldand(ptr, x) ({ __sync_fetch_and_and((ptr), (x)); }) #endif #ifndef pl_ldor #define pl_ldor(ptr, x) ({ __sync_fetch_and_or((ptr), (x)); }) #endif #ifndef pl_ldxor #define pl_ldxor(ptr, x) ({ __sync_fetch_and_xor((ptr), (x)); }) #endif #ifndef pl_ldsub #define pl_ldsub(ptr, x) ({ __sync_fetch_and_sub((ptr), (x)); }) #endif /* certain _noret operations may be defined from the regular ones */ #if !defined(pl_inc_noret) && defined(pl_inc) # define pl_inc_noret(ptr) (void)pl_inc(ptr) #endif #if !defined(pl_dec_noret) && defined(pl_dec) # define pl_dec_noret(ptr) (void)pl_dec(ptr) #endif #if !defined(pl_add_noret) && defined(pl_add) # define pl_add_noret(ptr, x) (void)pl_add(ptr, x) #endif #if !defined(pl_sub_noret) && defined(pl_sub) # define pl_sub_noret(ptr, x) (void)pl_sub(ptr, x) #endif #if !defined(pl_or_noret) && defined(pl_or) # define pl_or_noret(ptr, x) (void)pl_or(ptr, x) #endif #if !defined(pl_and_noret) && defined(pl_and) # define pl_and_noret(ptr, x) (void)pl_and(ptr, x) #endif #if !defined(pl_xor_noret) && defined(pl_xor) # define pl_xor_noret(ptr, x) (void)pl_xor(ptr, x) #endif /* certain memory orders may fallback to the generic seq_cst definition */ #if !defined(pl_load_lax) && defined(pl_load) #define pl_load_lax pl_load #endif #if !defined(pl_store_lax) && defined(pl_store) #define pl_store_lax pl_store #endif #if !defined(pl_inc_lax) && defined(pl_inc) # define pl_inc_lax pl_inc #endif #if !defined(pl_inc_acq) && defined(pl_inc) # define pl_inc_acq pl_inc #endif #if !defined(pl_inc_rel) && defined(pl_inc) # define pl_inc_rel pl_inc #endif #if !defined(pl_dec_lax) && defined(pl_dec) # define pl_dec_lax pl_dec #endif #if !defined(pl_dec_acq) && defined(pl_dec) # define pl_dec_acq pl_dec #endif #if !defined(pl_dec_rel) && defined(pl_dec) # define pl_dec_rel pl_dec #endif #if !defined(pl_inc_noret_lax) && defined(pl_inc_noret) # define pl_inc_noret_lax pl_inc_noret #endif #if !defined(pl_inc_noret_acq) && defined(pl_inc_noret) # define pl_inc_noret_acq pl_inc_noret #endif #if !defined(pl_inc_noret_rel) && defined(pl_inc_noret) # define pl_inc_noret_rel pl_inc_noret #endif #if !defined(pl_dec_noret_lax) && defined(pl_dec_noret) # define pl_dec_noret_lax pl_dec_noret #endif #if !defined(pl_dec_noret_acq) && defined(pl_dec_noret) # define pl_dec_noret_acq pl_dec_noret #endif #if !defined(pl_dec_noret_rel) && defined(pl_dec_noret) # define pl_dec_noret_rel pl_dec_noret #endif #if !defined(pl_add_lax) && defined(pl_add) # define pl_add_lax pl_add #endif #if !defined(pl_add_acq) && defined(pl_add) # define pl_add_acq pl_add #endif #if !defined(pl_add_rel) && defined(pl_add) # define pl_add_rel pl_add #endif #if !defined(pl_sub_lax) && defined(pl_sub) # define pl_sub_lax pl_sub #endif #if !defined(pl_sub_acq) && defined(pl_sub) # define pl_sub_acq pl_sub #endif #if !defined(pl_sub_rel) && defined(pl_sub) # define pl_sub_rel pl_sub #endif #if !defined(pl_and_lax) && defined(pl_and) # define pl_and_lax pl_and #endif #if !defined(pl_and_acq) && defined(pl_and) # define pl_and_acq pl_and #endif #if !defined(pl_and_rel) && defined(pl_and) # define pl_and_rel pl_and #endif #if !defined(pl_or_lax) && defined(pl_or) # define pl_or_lax pl_or #endif #if !defined(pl_or_acq) && defined(pl_or) # define pl_or_acq pl_or #endif #if !defined(pl_or_rel) && defined(pl_or) # define pl_or_rel pl_or #endif #if !defined(pl_xor_lax) && defined(pl_xor) # define pl_xor_lax pl_xor #endif #if !defined(pl_xor_acq) && defined(pl_xor) # define pl_xor_acq pl_xor #endif #if !defined(pl_xor_rel) && defined(pl_xor) # define pl_xor_rel pl_xor #endif #if !defined(pl_add_noret_lax) && defined(pl_add_noret) # define pl_add_noret_lax pl_add_noret #endif #if !defined(pl_add_noret_acq) && defined(pl_add_noret) # define pl_add_noret_acq pl_add_noret #endif #if !defined(pl_add_noret_rel) && defined(pl_add_noret) # define pl_add_noret_rel pl_add_noret #endif #if !defined(pl_sub_noret_lax) && defined(pl_sub_noret) # define pl_sub_noret_lax pl_sub_noret #endif #if !defined(pl_sub_noret_acq) && defined(pl_sub_noret) # define pl_sub_noret_acq pl_sub_noret #endif #if !defined(pl_sub_noret_rel) && defined(pl_sub_noret) # define pl_sub_noret_rel pl_sub_noret #endif #if !defined(pl_and_noret_lax) && defined(pl_and_noret) # define pl_and_noret_lax pl_and_noret #endif #if !defined(pl_and_noret_acq) && defined(pl_and_noret) # define pl_and_noret_acq pl_and_noret #endif #if !defined(pl_and_noret_rel) && defined(pl_and_noret) # define pl_and_noret_rel pl_and_noret #endif #if !defined(pl_or_noret_lax) && defined(pl_or_noret) # define pl_or_noret_lax pl_or_noret #endif #if !defined(pl_or_noret_acq) && defined(pl_or_noret) # define pl_or_noret_acq pl_or_noret #endif #if !defined(pl_or_noret_rel) && defined(pl_or_noret) # define pl_or_noret_rel pl_or_noret #endif #if !defined(pl_xor_noret_lax) && defined(pl_xor_noret) # define pl_xor_noret_lax pl_xor_noret #endif #if !defined(pl_xor_noret_acq) && defined(pl_xor_noret) # define pl_xor_noret_acq pl_xor_noret #endif #if !defined(pl_xor_noret_rel) && defined(pl_xor_noret) # define pl_xor_noret_rel pl_xor_noret #endif #if !defined(pl_btr_lax) && defined(pl_btr) # define pl_btr_lax pl_btr #endif #if !defined(pl_btr_acq) && defined(pl_btr) # define pl_btr_acq pl_btr #endif #if !defined(pl_btr_rel) && defined(pl_btr) # define pl_btr_rel pl_btr #endif #if !defined(pl_bts_lax) && defined(pl_bts) # define pl_bts_lax pl_bts #endif #if !defined(pl_bts_acq) && defined(pl_bts) # define pl_bts_acq pl_bts #endif #if !defined(pl_bts_rel) && defined(pl_bts) # define pl_bts_rel pl_bts #endif #if !defined(pl_ldadd_lax) && defined(pl_ldadd) # define pl_ldadd_lax pl_ldadd #endif #if !defined(pl_ldadd_acq) && defined(pl_ldadd) # define pl_ldadd_acq pl_ldadd #endif #if !defined(pl_ldadd_rel) && defined(pl_ldadd) # define pl_ldadd_rel pl_ldadd #endif #if !defined(pl_ldsub_lax) && defined(pl_ldsub) # define pl_ldsub_lax pl_ldsub #endif #if !defined(pl_ldsub_acq) && defined(pl_ldsub) # define pl_ldsub_acq pl_ldsub #endif #if !defined(pl_ldsub_rel) && defined(pl_ldsub) # define pl_ldsub_rel pl_ldsub #endif #if !defined(pl_ldand_lax) && defined(pl_ldand) # define pl_ldand_lax pl_ldand #endif #if !defined(pl_ldand_acq) && defined(pl_ldand) # define pl_ldand_acq pl_ldand #endif #if !defined(pl_ldand_rel) && defined(pl_ldand) # define pl_ldand_rel pl_ldand #endif #if !defined(pl_ldor_lax) && defined(pl_ldor) # define pl_ldor_lax pl_ldor #endif #if !defined(pl_ldor_acq) && defined(pl_ldor) # define pl_ldor_acq pl_ldor #endif #if !defined(pl_ldor_rel) && defined(pl_ldor) # define pl_ldor_rel pl_ldor #endif #if !defined(pl_ldxor_lax) && defined(pl_ldxor) # define pl_ldxor_lax pl_ldxor #endif #if !defined(pl_ldxor_acq) && defined(pl_ldxor) # define pl_ldxor_acq pl_ldxor #endif #if !defined(pl_ldxor_rel) && defined(pl_ldxor) # define pl_ldxor_rel pl_ldxor #endif #endif /* PL_ATOMIC_OPS_H */