summaryrefslogtreecommitdiffstats
path: root/include/gcc/x86
diff options
context:
space:
mode:
Diffstat (limited to 'include/gcc/x86')
-rw-r--r--include/gcc/x86/ck_pr.h157
1 files changed, 93 insertions, 64 deletions
diff --git a/include/gcc/x86/ck_pr.h b/include/gcc/x86/ck_pr.h
index a04cebf..5194dee 100644
--- a/include/gcc/x86/ck_pr.h
+++ b/include/gcc/x86/ck_pr.h
@@ -45,15 +45,9 @@
/* Minimum requirements for the CK_PR interface are met. */
#define CK_F_PR
-#ifdef CK_MD_UMP
-#define CK_PR_LOCK_PREFIX
-#else
-#define CK_PR_LOCK_PREFIX "lock "
-#endif
-
/*
- * Prevent speculative execution in busy-wait loops (P4 <=)
- * or "predefined delay".
+ * Prevent speculative execution in busy-wait loops (P4 <=) or "predefined
+ * delay".
*/
CK_CC_INLINE static void
ck_pr_stall(void)
@@ -62,28 +56,52 @@ ck_pr_stall(void)
return;
}
+#ifdef CK_MD_UMP
+#define CK_PR_LOCK_PREFIX
+#define CK_PR_FENCE(T, I) \
+ CK_CC_INLINE static void \
+ ck_pr_fence_strict_##T(void) \
+ { \
+ __asm__ __volatile__("" ::: "memory"); \
+ return; \
+ }
+#else
+#define CK_PR_LOCK_PREFIX "lock "
#define CK_PR_FENCE(T, I) \
CK_CC_INLINE static void \
ck_pr_fence_strict_##T(void) \
{ \
__asm__ __volatile__(I ::: "memory"); \
+ return; \
}
+#endif /* CK_MD_UMP */
-CK_PR_FENCE(atomic, "sfence")
-CK_PR_FENCE(atomic_store, "sfence")
-CK_PR_FENCE(atomic_load, "mfence")
-CK_PR_FENCE(store_atomic, "sfence")
-CK_PR_FENCE(load_atomic, "mfence")
-CK_PR_FENCE(load, "lfence")
-CK_PR_FENCE(load_store, "mfence")
-CK_PR_FENCE(store, "sfence")
-CK_PR_FENCE(store_load, "mfence")
-CK_PR_FENCE(memory, "mfence")
-CK_PR_FENCE(release, "mfence")
-CK_PR_FENCE(acquire, "mfence")
-CK_PR_FENCE(acqrel, "mfence")
-CK_PR_FENCE(lock, "mfence")
-CK_PR_FENCE(unlock, "mfence")
+#if defined(CK_MD_SSE_DISABLE)
+/* If SSE is disabled, then use atomic operations for serialization. */
+#define CK_MD_X86_MFENCE "lock addl $0, (%%esp)"
+#define CK_MD_X86_SFENCE CK_MD_X86_MFENCE
+#define CK_MD_X86_LFENCE CK_MD_X86_MFENCE
+#else
+#define CK_MD_X86_SFENCE "sfence"
+#define CK_MD_X86_LFENCE "lfence"
+#define CK_MD_X86_MFENCE "mfence"
+#endif /* !CK_MD_SSE_DISABLE */
+
+CK_PR_FENCE(atomic, "")
+CK_PR_FENCE(atomic_store, "")
+CK_PR_FENCE(atomic_load, "")
+CK_PR_FENCE(store_atomic, "")
+CK_PR_FENCE(load_atomic, "")
+CK_PR_FENCE(load, CK_MD_X86_LFENCE)
+CK_PR_FENCE(load_store, CK_MD_X86_MFENCE)
+CK_PR_FENCE(store, CK_MD_X86_SFENCE)
+CK_PR_FENCE(store_load, CK_MD_X86_MFENCE)
+CK_PR_FENCE(memory, CK_MD_X86_MFENCE)
+CK_PR_FENCE(release, CK_MD_X86_MFENCE)
+CK_PR_FENCE(acquire, CK_MD_X86_MFENCE)
+CK_PR_FENCE(acqrel, CK_MD_X86_MFENCE)
+CK_PR_FENCE(lock, CK_MD_X86_MFENCE)
+CK_PR_FENCE(unlock, CK_MD_X86_MFENCE)
#undef CK_PR_FENCE
@@ -215,18 +233,18 @@ CK_PR_FAA_S(8, uint8_t, "xaddb")
}
#define CK_PR_UNARY_V(K, S, T, C, I) \
- CK_CC_INLINE static void \
- ck_pr_##K##_##S##_zero(T *target, bool *r) \
+ CK_CC_INLINE static bool \
+ ck_pr_##K##_##S##_is_zero(T *target) \
{ \
+ bool ret; \
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \
: "+m" (*(C *)target), \
- "=m" (*r) \
+ "=qm" (ret) \
: \
: "memory", "cc"); \
- return; \
+ return ret; \
}
-
#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I)
#define CK_PR_GENERATE(K) \
@@ -289,8 +307,38 @@ CK_PR_GENERATE(xor)
#undef CK_PR_BINARY
/*
- * Atomic compare and swap.
+ * Atomic compare and swap, with a variant that sets *v to the old value of target.
*/
+#ifdef __GCC_ASM_FLAG_OUTPUTS__
+#define CK_PR_CAS(S, M, T, C, I) \
+ CK_CC_INLINE static bool \
+ ck_pr_cas_##S(M *target, T compare, T set) \
+ { \
+ bool z; \
+ __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %3, %0" \
+ : "+m" (*(C *)target), \
+ "=@ccz" (z), \
+ /* RAX is clobbered by cmpxchg. */ \
+ "+a" (compare) \
+ : "q" (set) \
+ : "memory", "cc"); \
+ return z; \
+ } \
+ \
+ CK_CC_INLINE static bool \
+ ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
+ { \
+ bool z; \
+ __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %3, %0;" \
+ : "+m" (*(C *)target), \
+ "=@ccz" (z), \
+ "+a" (compare) \
+ : "q" (set) \
+ : "memory", "cc"); \
+ *(T *)v = compare; \
+ return z; \
+ }
+#else
#define CK_PR_CAS(S, M, T, C, I) \
CK_CC_INLINE static bool \
ck_pr_cas_##S(M *target, T compare, T set) \
@@ -303,7 +351,23 @@ CK_PR_GENERATE(xor)
"a" (compare) \
: "memory", "cc"); \
return z; \
+ } \
+ \
+ CK_CC_INLINE static bool \
+ ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
+ { \
+ bool z; \
+ __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %3, %0;" \
+ "setz %1;" \
+ : "+m" (*(C *)target), \
+ "=q" (z), \
+ "+a" (compare) \
+ : "q" (set) \
+ : "memory", "cc"); \
+ *(T *)v = compare; \
+ return z; \
}
+#endif
CK_PR_CAS(ptr, void, void *, char, "cmpxchgl")
@@ -320,41 +384,6 @@ CK_PR_CAS_S(8, uint8_t, "cmpxchgb")
#undef CK_PR_CAS
/*
- * Compare and swap, set *v to old value of target.
- */
-#define CK_PR_CAS_O(S, M, T, C, I, R) \
- CK_CC_INLINE static bool \
- ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \
- { \
- bool z; \
- __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \
- "mov %% " R ", %2;" \
- "setz %1;" \
- : "+m" (*(C *)target), \
- "=a" (z), \
- "=m" (*(C *)v) \
- : "q" (set), \
- "a" (compare) \
- : "memory", "cc"); \
- return (bool)z; \
- }
-
-CK_PR_CAS_O(ptr, void, void *, char, "l", "eax")
-
-#define CK_PR_CAS_O_S(S, T, I, R) \
- CK_PR_CAS_O(S, T, T, T, I, R)
-
-CK_PR_CAS_O_S(char, char, "b", "al")
-CK_PR_CAS_O_S(int, int, "l", "eax")
-CK_PR_CAS_O_S(uint, unsigned int, "l", "eax")
-CK_PR_CAS_O_S(32, uint32_t, "l", "eax")
-CK_PR_CAS_O_S(16, uint16_t, "w", "ax")
-CK_PR_CAS_O_S(8, uint8_t, "b", "al")
-
-#undef CK_PR_CAS_O_S
-#undef CK_PR_CAS_O
-
-/*
* Atomic bit test operations.
*/
#define CK_PR_BT(K, S, T, P, C, I) \