diff options
Diffstat (limited to '')
64 files changed, 14114 insertions, 0 deletions
diff --git a/include/ck_array.h b/include/ck_array.h new file mode 100644 index 0000000..9cb97b2 --- /dev/null +++ b/include/ck_array.h @@ -0,0 +1,100 @@ +/* + * Copyright 2013-2015 Samy Al Bahra + * Copyright 2013-2014 AppNexus, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_ARRAY_H +#define CK_ARRAY_H + +#include <ck_cc.h> +#include <ck_malloc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct _ck_array { + unsigned int n_committed; + unsigned int length; + void *values[]; +}; + +struct ck_array { + struct ck_malloc *allocator; + struct _ck_array *active; + unsigned int n_entries; + struct _ck_array *transaction; +}; +typedef struct ck_array ck_array_t; + +struct ck_array_iterator { + struct _ck_array *snapshot; +}; +typedef struct ck_array_iterator ck_array_iterator_t; + +#define CK_ARRAY_MODE_SPMC 0U +#define CK_ARRAY_MODE_MPMC (void) /* Unsupported. */ + +bool ck_array_init(ck_array_t *, unsigned int, struct ck_malloc *, unsigned int); +bool ck_array_commit(ck_array_t *); +bool ck_array_put(ck_array_t *, void *); +int ck_array_put_unique(ck_array_t *, void *); +bool ck_array_remove(ck_array_t *, void *); +void ck_array_deinit(ck_array_t *, bool); + +CK_CC_INLINE static unsigned int +ck_array_length(struct ck_array *array) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + return ck_pr_load_uint(&a->n_committed); +} + +CK_CC_INLINE static void * +ck_array_buffer(struct ck_array *array, unsigned int *length) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + *length = ck_pr_load_uint(&a->n_committed); + return a->values; +} + +CK_CC_INLINE static bool +ck_array_initialized(struct ck_array *array) +{ + + return ck_pr_load_ptr(&array->active) != NULL; +} + +#define CK_ARRAY_FOREACH(a, i, b) \ + (i)->snapshot = ck_pr_load_ptr(&(a)->active); \ + ck_pr_fence_load(); \ + for (unsigned int _ck_i = 0; \ + _ck_i < (a)->active->n_committed && \ + ((*b) = (a)->active->values[_ck_i], 1); \ + _ck_i++) + +#endif /* CK_ARRAY_H */ diff --git a/include/ck_backoff.h b/include/ck_backoff.h new file mode 100644 index 0000000..82a4f21 --- /dev/null +++ b/include/ck_backoff.h @@ -0,0 +1,57 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BACKOFF_H +#define CK_BACKOFF_H + +#include <ck_cc.h> +#include <ck_pr.h> + +#ifndef CK_BACKOFF_CEILING +#define CK_BACKOFF_CEILING ((1 << 20) - 1) +#endif + +#define CK_BACKOFF_INITIALIZER (1 << 9) + +typedef unsigned int ck_backoff_t; + +/* + * This is a exponential back-off implementation. + */ +CK_CC_INLINE static void +ck_backoff_eb(unsigned int *c) +{ + unsigned int i, ceiling; + + ceiling = *c; + for (i = 0; i < ceiling; i++) + ck_pr_barrier(); + + *c = ceiling <<= ceiling < CK_BACKOFF_CEILING; + return; +} + +#endif /* CK_BACKOFF_H */ diff --git a/include/ck_barrier.h b/include/ck_barrier.h new file mode 100644 index 0000000..d4c12ca --- /dev/null +++ b/include/ck_barrier.h @@ -0,0 +1,164 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BARRIER_H +#define CK_BARRIER_H + +#include <ck_spinlock.h> + +struct ck_barrier_centralized { + unsigned int value; + unsigned int sense; +}; +typedef struct ck_barrier_centralized ck_barrier_centralized_t; + +struct ck_barrier_centralized_state { + unsigned int sense; +}; +typedef struct ck_barrier_centralized_state ck_barrier_centralized_state_t; + +#define CK_BARRIER_CENTRALIZED_INITIALIZER {0, 0} +#define CK_BARRIER_CENTRALIZED_STATE_INITIALIZER {0} + +void ck_barrier_centralized(ck_barrier_centralized_t *, + ck_barrier_centralized_state_t *, unsigned int); + +struct ck_barrier_combining_group { + unsigned int k; + unsigned int count; + unsigned int sense; + struct ck_barrier_combining_group *parent; + struct ck_barrier_combining_group *left; + struct ck_barrier_combining_group *right; + struct ck_barrier_combining_group *next; +} CK_CC_CACHELINE; +typedef struct ck_barrier_combining_group ck_barrier_combining_group_t; + +struct ck_barrier_combining_state { + unsigned int sense; +}; +typedef struct ck_barrier_combining_state ck_barrier_combining_state_t; + +#define CK_BARRIER_COMBINING_STATE_INITIALIZER {~0} + +struct ck_barrier_combining { + struct ck_barrier_combining_group *root; + ck_spinlock_fas_t mutex; +}; +typedef struct ck_barrier_combining ck_barrier_combining_t; + +void ck_barrier_combining_init(ck_barrier_combining_t *, ck_barrier_combining_group_t *); + +void ck_barrier_combining_group_init(ck_barrier_combining_t *, + ck_barrier_combining_group_t *, unsigned int); + +void ck_barrier_combining(ck_barrier_combining_t *, + ck_barrier_combining_group_t *, + ck_barrier_combining_state_t *); + +struct ck_barrier_dissemination_flag { + unsigned int tflag; + unsigned int *pflag; +}; +typedef struct ck_barrier_dissemination_flag ck_barrier_dissemination_flag_t; + +struct ck_barrier_dissemination { + unsigned int nthr; + unsigned int size; + unsigned int tid; + struct ck_barrier_dissemination_flag *flags[2]; +}; +typedef struct ck_barrier_dissemination ck_barrier_dissemination_t; + +struct ck_barrier_dissemination_state { + int parity; + unsigned int sense; + unsigned int tid; +}; +typedef struct ck_barrier_dissemination_state ck_barrier_dissemination_state_t; + +void ck_barrier_dissemination_init(ck_barrier_dissemination_t *, + ck_barrier_dissemination_flag_t **, unsigned int); + +void ck_barrier_dissemination_subscribe(ck_barrier_dissemination_t *, + ck_barrier_dissemination_state_t *); + +unsigned int ck_barrier_dissemination_size(unsigned int); + +void ck_barrier_dissemination(ck_barrier_dissemination_t *, + ck_barrier_dissemination_state_t *); + +struct ck_barrier_tournament_round { + int role; + unsigned int *opponent; + unsigned int flag; +}; +typedef struct ck_barrier_tournament_round ck_barrier_tournament_round_t; + +struct ck_barrier_tournament { + unsigned int tid; + unsigned int size; + struct ck_barrier_tournament_round **rounds; +}; +typedef struct ck_barrier_tournament ck_barrier_tournament_t; + +struct ck_barrier_tournament_state { + unsigned int sense; + unsigned int vpid; +}; +typedef struct ck_barrier_tournament_state ck_barrier_tournament_state_t; + +void ck_barrier_tournament_subscribe(ck_barrier_tournament_t *, + ck_barrier_tournament_state_t *); +void ck_barrier_tournament_init(ck_barrier_tournament_t *, + ck_barrier_tournament_round_t **, + unsigned int); +unsigned int ck_barrier_tournament_size(unsigned int); +void ck_barrier_tournament(ck_barrier_tournament_t *, ck_barrier_tournament_state_t *); + +struct ck_barrier_mcs { + unsigned int tid; + unsigned int *children[2]; + unsigned int childnotready[4]; + unsigned int dummy; + unsigned int havechild[4]; + unsigned int *parent; + unsigned int parentsense; +}; +typedef struct ck_barrier_mcs ck_barrier_mcs_t; + +struct ck_barrier_mcs_state { + unsigned int sense; + unsigned int vpid; +}; +typedef struct ck_barrier_mcs_state ck_barrier_mcs_state_t; + +void ck_barrier_mcs_init(ck_barrier_mcs_t *, unsigned int); +void ck_barrier_mcs_subscribe(ck_barrier_mcs_t *, ck_barrier_mcs_state_t *); +void ck_barrier_mcs(ck_barrier_mcs_t *, ck_barrier_mcs_state_t *); + +#endif /* CK_BARRIER_H */ diff --git a/include/ck_bitmap.h b/include/ck_bitmap.h new file mode 100644 index 0000000..624e953 --- /dev/null +++ b/include/ck_bitmap.h @@ -0,0 +1,515 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * Copyright 2012-2014 AppNexus, Inc. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BITMAP_H +#define CK_BITMAP_H + +#include <ck_cc.h> +#include <ck_limits.h> +#include <ck_pr.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> +#include <ck_string.h> + +#if !defined(CK_F_PR_LOAD_UINT) || !defined(CK_F_PR_STORE_UINT) || \ + !defined(CK_F_PR_AND_UINT) || !defined(CK_F_PR_OR_UINT) || \ + !defined(CK_F_CC_CTZ) +#error "ck_bitmap is not supported on your platform." +#endif + +#define CK_BITMAP_BLOCK (sizeof(unsigned int) * CHAR_BIT) +#define CK_BITMAP_OFFSET(i) ((i) % CK_BITMAP_BLOCK) +#define CK_BITMAP_BIT(i) (1U << CK_BITMAP_OFFSET(i)) +#define CK_BITMAP_PTR(x, i) ((x) + ((i) / CK_BITMAP_BLOCK)) +#define CK_BITMAP_BLOCKS(n) (((n) + CK_BITMAP_BLOCK - 1) / CK_BITMAP_BLOCK) + +#define CK_BITMAP_INSTANCE(n_entries) \ + union { \ + struct { \ + unsigned int n_bits; \ + unsigned int map[CK_BITMAP_BLOCKS(n_entries)]; \ + } content; \ + struct ck_bitmap bitmap; \ + } + +#define CK_BITMAP_ITERATOR_INIT(a, b) \ + ck_bitmap_iterator_init((a), &(b)->bitmap) + +#define CK_BITMAP_INIT(a, b, c) \ + ck_bitmap_init(&(a)->bitmap, (b), (c)) + +#define CK_BITMAP_NEXT(a, b, c) \ + ck_bitmap_next(&(a)->bitmap, (b), (c)) + +#define CK_BITMAP_SET(a, b) \ + ck_bitmap_set(&(a)->bitmap, (b)) + +#define CK_BITMAP_BTS(a, b) \ + ck_bitmap_bts(&(a)->bitmap, (b)) + +#define CK_BITMAP_RESET(a, b) \ + ck_bitmap_reset(&(a)->bitmap, (b)) + +#define CK_BITMAP_TEST(a, b) \ + ck_bitmap_test(&(a)->bitmap, (b)) + +#define CK_BITMAP_UNION(a, b) \ + ck_bitmap_union(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_INTERSECTION(a, b) \ + ck_bitmap_intersection(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_INTERSECTION_NEGATE(a, b) \ + ck_bitmap_intersection_negate(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_CLEAR(a) \ + ck_bitmap_clear(&(a)->bitmap) + +#define CK_BITMAP_EMPTY(a, b) \ + ck_bitmap_empty(&(a)->bitmap, b) + +#define CK_BITMAP_FULL(a, b) \ + ck_bitmap_full(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT(a, b) \ + ck_bitmap_count(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT_INTERSECT(a, b, c) \ + ck_bitmap_count_intersect(&(a)->bitmap, b, c) + +#define CK_BITMAP_BITS(a) \ + ck_bitmap_bits(&(a)->bitmap) + +#define CK_BITMAP_BUFFER(a) \ + ck_bitmap_buffer(&(a)->bitmap) + +#define CK_BITMAP(a) \ + (&(a)->bitmap) + +struct ck_bitmap { + unsigned int n_bits; + unsigned int map[]; +}; +typedef struct ck_bitmap ck_bitmap_t; + +struct ck_bitmap_iterator { + unsigned int cache; + unsigned int n_block; + unsigned int n_limit; +}; +typedef struct ck_bitmap_iterator ck_bitmap_iterator_t; + +CK_CC_INLINE static unsigned int +ck_bitmap_base(unsigned int n_bits) +{ + + return CK_BITMAP_BLOCKS(n_bits) * sizeof(unsigned int); +} + +/* + * Returns the required number of bytes for a ck_bitmap_t object supporting the + * specified number of bits. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_size(unsigned int n_bits) +{ + + return ck_bitmap_base(n_bits) + sizeof(struct ck_bitmap); +} + +/* + * Returns total number of bits in specified bitmap. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_bits(const struct ck_bitmap *bitmap) +{ + + return bitmap->n_bits; +} + +/* + * Returns a pointer to the bit buffer associated + * with the specified bitmap. + */ +CK_CC_INLINE static void * +ck_bitmap_buffer(struct ck_bitmap *bitmap) +{ + + return bitmap->map; +} + +/* + * Sets the bit at the offset specified in the second argument. + */ +CK_CC_INLINE static void +ck_bitmap_set(struct ck_bitmap *bitmap, unsigned int n) +{ + + ck_pr_or_uint(CK_BITMAP_PTR(bitmap->map, n), CK_BITMAP_BIT(n)); + return; +} + +/* + * Performs a test-and-set operation at the offset specified in the + * second argument. + * Returns true if the bit at the specified offset was already set, + * false otherwise. + */ +CK_CC_INLINE static bool +ck_bitmap_bts(struct ck_bitmap *bitmap, unsigned int n) +{ + + return ck_pr_bts_uint(CK_BITMAP_PTR(bitmap->map, n), + CK_BITMAP_OFFSET(n)); +} + +/* + * Resets the bit at the offset specified in the second argument. + */ +CK_CC_INLINE static void +ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n) +{ + + ck_pr_and_uint(CK_BITMAP_PTR(bitmap->map, n), ~CK_BITMAP_BIT(n)); + return; +} + +/* + * Determines whether the bit at offset specified in the + * second argument is set. + */ +CK_CC_INLINE static bool +ck_bitmap_test(const struct ck_bitmap *bitmap, unsigned int n) +{ + unsigned int block; + + block = ck_pr_load_uint(CK_BITMAP_PTR(bitmap->map, n)); + return block & CK_BITMAP_BIT(n); +} + +/* + * Combines bits from second bitmap into the first bitmap. This is not a + * linearized operation with respect to the complete bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_union(struct ck_bitmap *dst, const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_buckets = dst->n_bits; + + if (src->n_bits < dst->n_bits) + n_buckets = src->n_bits; + + n_buckets = CK_BITMAP_BLOCKS(n_buckets); + for (n = 0; n < n_buckets; n++) { + ck_pr_or_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } + + return; +} + +/* + * Intersects bits from second bitmap into the first bitmap. This is + * not a linearized operation with respect to the complete bitmap. + * Any trailing bit in dst is cleared. + */ +CK_CC_INLINE static void +ck_bitmap_intersection(struct ck_bitmap *dst, const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_buckets = dst->n_bits; + unsigned int n_intersect = n_buckets; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_buckets = CK_BITMAP_BLOCKS(n_buckets); + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } + + for (; n < n_buckets; n++) + ck_pr_store_uint(&dst->map[n], 0); + + return; +} + +/* + * Intersects the complement of bits from second bitmap into the first + * bitmap. This is not a linearized operation with respect to the + * complete bitmap. Any trailing bit in dst is left as is. + */ +CK_CC_INLINE static void +ck_bitmap_intersection_negate(struct ck_bitmap *dst, + const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_intersect = dst->n_bits; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + (~ck_pr_load_uint(&src->map[n]))); + } + + return; +} + +/* + * Resets all bits in the provided bitmap. This is not a linearized + * operation in ck_bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_clear(struct ck_bitmap *bitmap) +{ + unsigned int i; + unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / + sizeof(unsigned int); + + for (i = 0; i < n_buckets; i++) + ck_pr_store_uint(&bitmap->map[i], 0); + + return; +} + +/* + * Returns true if the first limit bits in bitmap are cleared. If + * limit is greater than the bitmap size, limit is truncated to that + * size. + */ +CK_CC_INLINE static bool +ck_bitmap_empty(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int i, words, slop; + + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != 0) { + return false; + } + } + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } + + return true; +} + +/* + * Returns true if the first limit bits in bitmap are set. If limit + * is greater than the bitmap size, limit is truncated to that size. + */ +CK_CC_UNUSED static bool +ck_bitmap_full(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int i, slop, words; + + if (limit > bitmap->n_bits) { + limit = bitmap->n_bits; + } + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != -1U) + return false; + } + + if (slop > 0) { + unsigned int word; + + word = ~ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } + return true; +} + +/* + * Returns the number of set bit in bitmap, upto (and excluding) + * limit. If limit is greater than the bitmap size, it is truncated + * to that size. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_count(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int count, i, slop, words; + + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) + count += ck_cc_popcount(ck_pr_load_uint(&bitmap->map[i])); + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; +} + +/* + * Returns the number of set bit in the intersection of two bitmaps, + * upto (and excluding) limit. If limit is greater than either bitmap + * size, it is truncated to the smallest. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_count_intersect(const ck_bitmap_t *x, const ck_bitmap_t *y, + unsigned int limit) +{ + unsigned int count, i, slop, words; + + if (limit > x->n_bits) + limit = x->n_bits; + + if (limit > y->n_bits) + limit = y->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) { + unsigned int xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + count += ck_cc_popcount(xi & yi); + } + + if (slop > 0) { + unsigned int word, xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + word = xi & yi; + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; +} + +/* + * Initializes a ck_bitmap pointing to a region of memory with + * ck_bitmap_size(n_bits) bytes. Third argument determines whether + * default bit value is 1 (true) or 0 (false). + */ +CK_CC_INLINE static void +ck_bitmap_init(struct ck_bitmap *bitmap, + unsigned int n_bits, + bool set) +{ + unsigned int base = ck_bitmap_base(n_bits); + + bitmap->n_bits = n_bits; + memset(bitmap->map, -(int)set, base); + + if (set == true) { + unsigned int b = n_bits % CK_BITMAP_BLOCK; + + if (b == 0) + return; + + *CK_BITMAP_PTR(bitmap->map, n_bits - 1) &= (1U << b) - 1U; + } + + return; +} + +/* + * Initialize iterator for use with provided bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, + const struct ck_bitmap *bitmap) +{ + + i->n_block = 0; + i->n_limit = CK_BITMAP_BLOCKS(bitmap->n_bits); + if (i->n_limit > 0) { + i->cache = ck_pr_load_uint(&bitmap->map[0]); + } else { + i->cache = 0; + } + return; +} + +/* + * Iterate to next bit. + */ +CK_CC_INLINE static bool +ck_bitmap_next(const struct ck_bitmap *bitmap, + struct ck_bitmap_iterator *i, + unsigned int *bit) +{ + unsigned int cache = i->cache; + unsigned int n_block = i->n_block; + unsigned int n_limit = i->n_limit; + + if (cache == 0) { + if (n_block >= n_limit) + return false; + + for (n_block++; n_block < n_limit; n_block++) { + cache = ck_pr_load_uint(&bitmap->map[n_block]); + if (cache != 0) + goto non_zero; + } + + i->cache = 0; + i->n_block = n_block; + return false; + } + +non_zero: + *bit = CK_BITMAP_BLOCK * n_block + ck_cc_ctz(cache); + i->cache = cache & (cache - 1); + i->n_block = n_block; + return true; +} + +#endif /* CK_BITMAP_H */ diff --git a/include/ck_brlock.h b/include/ck_brlock.h new file mode 100644 index 0000000..d1b4ed1 --- /dev/null +++ b/include/ck_brlock.h @@ -0,0 +1,279 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BRLOCK_H +#define CK_BRLOCK_H + +/* + * Big reader spinlocks provide cache-local contention-free read + * lock acquisition in the absence of writers. This comes at the + * cost of O(n) write lock acquisition. They were first implemented + * in the Linux kernel by Ingo Molnar and David S. Miller around the + * year 2000. + * + * This implementation is thread-agnostic which comes at the cost + * of larger reader objects due to necessary linkage overhead. In + * order to cut down on TLB pressure, it is recommended to allocate + * these objects on the same page. + */ + +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_brlock_reader { + unsigned int n_readers; + struct ck_brlock_reader *previous; + struct ck_brlock_reader *next; +}; +typedef struct ck_brlock_reader ck_brlock_reader_t; + +#define CK_BRLOCK_READER_INITIALIZER {0} + +struct ck_brlock { + struct ck_brlock_reader *readers; + unsigned int writer; +}; +typedef struct ck_brlock ck_brlock_t; + +#define CK_BRLOCK_INITIALIZER {NULL, false} + +CK_CC_INLINE static void +ck_brlock_init(struct ck_brlock *br) +{ + + br->readers = NULL; + br->writer = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_brlock_write_lock(struct ck_brlock *br) +{ + struct ck_brlock_reader *cursor; + + /* + * As the frequency of write acquisitions should be low, + * there is no point to more advanced contention avoidance. + */ + while (ck_pr_fas_uint(&br->writer, true) == true) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + /* The reader list is protected under the writer br. */ + for (cursor = br->readers; cursor != NULL; cursor = cursor->next) { + while (ck_pr_load_uint(&cursor->n_readers) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_brlock_write_unlock(struct ck_brlock *br) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&br->writer, false); + return; +} + +CK_CC_INLINE static bool +ck_brlock_write_trylock(struct ck_brlock *br, unsigned int factor) +{ + struct ck_brlock_reader *cursor; + unsigned int steps = 0; + + while (ck_pr_fas_uint(&br->writer, true) == true) { + if (++steps >= factor) + return false; + + ck_pr_stall(); + } + + /* + * We do not require a strict fence here as atomic RMW operations + * are serializing. + */ + ck_pr_fence_atomic_load(); + + for (cursor = br->readers; cursor != NULL; cursor = cursor->next) { + while (ck_pr_load_uint(&cursor->n_readers) != 0) { + if (++steps >= factor) { + ck_brlock_write_unlock(br); + return false; + } + + ck_pr_stall(); + } + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_brlock_read_register(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + reader->n_readers = 0; + reader->previous = NULL; + + /* Implicit compiler barrier. */ + ck_brlock_write_lock(br); + + reader->next = ck_pr_load_ptr(&br->readers); + if (reader->next != NULL) + reader->next->previous = reader; + ck_pr_store_ptr(&br->readers, reader); + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_unregister(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + ck_brlock_write_lock(br); + + if (reader->next != NULL) + reader->next->previous = reader->previous; + + if (reader->previous != NULL) + reader->previous->next = reader->next; + else + br->readers = reader->next; + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_lock(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + if (reader->n_readers >= 1) { + ck_pr_store_uint(&reader->n_readers, reader->n_readers + 1); + return; + } + + for (;;) { + while (ck_pr_load_uint(&br->writer) == true) + ck_pr_stall(); + +#if defined(__x86__) || defined(__x86_64__) + ck_pr_fas_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_store_load(); +#endif + + if (ck_pr_load_uint(&br->writer) == false) + break; + + ck_pr_store_uint(&reader->n_readers, 0); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_brlock_read_trylock(struct ck_brlock *br, + struct ck_brlock_reader *reader, + unsigned int factor) +{ + unsigned int steps = 0; + + if (reader->n_readers >= 1) { + ck_pr_store_uint(&reader->n_readers, reader->n_readers + 1); + return true; + } + + for (;;) { + while (ck_pr_load_uint(&br->writer) == true) { + if (++steps >= factor) + return false; + + ck_pr_stall(); + } + +#if defined(__x86__) || defined(__x86_64__) + ck_pr_fas_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_store_load(); +#endif + + if (ck_pr_load_uint(&br->writer) == false) + break; + + ck_pr_store_uint(&reader->n_readers, 0); + + if (++steps >= factor) + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_brlock_read_unlock(struct ck_brlock_reader *reader) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&reader->n_readers, reader->n_readers - 1); + return; +} + +#endif /* CK_BRLOCK_H */ diff --git a/include/ck_bytelock.h b/include/ck_bytelock.h new file mode 100644 index 0000000..d437316 --- /dev/null +++ b/include/ck_bytelock.h @@ -0,0 +1,196 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BYTELOCK_H +#define CK_BYTELOCK_H + +/* + * The implementations here are derived from the work described in: + * Dice, D. and Shavit, N. 2010. TLRW: return of the read-write lock. + * In Proceedings of the 22nd ACM Symposium on Parallelism in Algorithms + * and Architectures (Thira, Santorini, Greece, June 13 - 15, 2010). + * SPAA '10. ACM, New York, NY, 284-293. + */ + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> +#include <ck_limits.h> + +struct ck_bytelock { + unsigned int owner; + unsigned int n_readers; + uint8_t readers[CK_MD_CACHELINE - sizeof(unsigned int) * 2] CK_CC_ALIGN(8); +}; +typedef struct ck_bytelock ck_bytelock_t; + +#define CK_BYTELOCK_INITIALIZER { 0, 0, {0} } +#define CK_BYTELOCK_UNSLOTTED UINT_MAX + +CK_CC_INLINE static void +ck_bytelock_init(struct ck_bytelock *bytelock) +{ + unsigned int i; + + bytelock->owner = 0; + bytelock->n_readers = 0; + for (i = 0; i < sizeof bytelock->readers; i++) + bytelock->readers[i] = false; + + ck_pr_barrier(); + return; +} + +#ifdef CK_F_PR_LOAD_64 +#define CK_BYTELOCK_LENGTH sizeof(uint64_t) +#define CK_BYTELOCK_LOAD ck_pr_load_64 +#define CK_BYTELOCK_TYPE uint64_t +#elif defined(CK_F_PR_LOAD_32) +#define CK_BYTELOCK_LENGTH sizeof(uint32_t) +#define CK_BYTELOCK_LOAD ck_pr_load_32 +#define CK_BYTELOCK_TYPE uint32_t +#else +#error Unsupported platform. +#endif + +CK_CC_INLINE static void +ck_bytelock_write_lock(struct ck_bytelock *bytelock, unsigned int slot) +{ + CK_BYTELOCK_TYPE *readers = (void *)bytelock->readers; + unsigned int i; + + /* Announce upcoming writer acquisition. */ + while (ck_pr_cas_uint(&bytelock->owner, 0, slot) == false) + ck_pr_stall(); + + /* If we are slotted, we might be upgrading from a read lock. */ + if (slot <= sizeof bytelock->readers) + ck_pr_store_8(&bytelock->readers[slot - 1], false); + + /* + * Wait for slotted readers to drain out. This also provides the + * lock acquire semantics. + */ + ck_pr_fence_atomic_load(); + + for (i = 0; i < sizeof(bytelock->readers) / CK_BYTELOCK_LENGTH; i++) { + while (CK_BYTELOCK_LOAD(&readers[i]) != false) + ck_pr_stall(); + } + + /* Wait for unslotted readers to drain out. */ + while (ck_pr_load_uint(&bytelock->n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +#undef CK_BYTELOCK_LENGTH +#undef CK_BYTELOCK_LOAD +#undef CK_BYTELOCK_TYPE + +CK_CC_INLINE static void +ck_bytelock_write_unlock(struct ck_bytelock *bytelock) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&bytelock->owner, 0); + return; +} + +CK_CC_INLINE static void +ck_bytelock_read_lock(struct ck_bytelock *bytelock, unsigned int slot) +{ + + if (ck_pr_load_uint(&bytelock->owner) == slot) { + ck_pr_store_8(&bytelock->readers[slot - 1], true); + ck_pr_fence_strict_store(); + ck_pr_store_uint(&bytelock->owner, 0); + return; + } + + /* Unslotted threads will have to use the readers counter. */ + if (slot > sizeof bytelock->readers) { + for (;;) { + ck_pr_inc_uint(&bytelock->n_readers); + ck_pr_fence_atomic_load(); + if (ck_pr_load_uint(&bytelock->owner) == 0) + break; + ck_pr_dec_uint(&bytelock->n_readers); + + while (ck_pr_load_uint(&bytelock->owner) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; + } + + slot -= 1; + for (;;) { +#ifdef CK_F_PR_FAA_8 + ck_pr_fas_8(&bytelock->readers[slot], true); + ck_pr_fence_atomic_load(); +#else + ck_pr_store_8(&bytelock->readers[slot], true); + ck_pr_fence_store_load(); +#endif + + /* + * If there is no owner at this point, our slot has + * already been published and it is guaranteed no + * write acquisition will succeed until we drain out. + */ + if (ck_pr_load_uint(&bytelock->owner) == 0) + break; + + ck_pr_store_8(&bytelock->readers[slot], false); + while (ck_pr_load_uint(&bytelock->owner) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_bytelock_read_unlock(struct ck_bytelock *bytelock, unsigned int slot) +{ + + ck_pr_fence_unlock(); + + if (slot > sizeof bytelock->readers) + ck_pr_dec_uint(&bytelock->n_readers); + else + ck_pr_store_8(&bytelock->readers[slot - 1], false); + + return; +} + +#endif /* CK_BYTELOCK_H */ diff --git a/include/ck_cc.h b/include/ck_cc.h new file mode 100644 index 0000000..e17dc7b --- /dev/null +++ b/include/ck_cc.h @@ -0,0 +1,180 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_CC_H +#define CK_CC_H + +#if defined(__GNUC__) || defined(__SUNPRO_C) +#include "gcc/ck_cc.h" +#endif + +#ifndef CK_CC_RESTRICT +#define CK_CC_RESTRICT +#endif + +#ifndef CK_CC_INLINE +#define CK_CC_INLINE inline +#endif + +#ifndef CK_CC_FORCE_INLINE +#define CK_CC_FORCE_INLINE inline +#endif + +#define CK_CC_DECONST_PTR(X) ((void *)(uintptr_t)(X)) + +/* + * Container function. + * This relies on (compiler) implementation-defined behavior. + */ +#define CK_CC_CONTAINER(F, T, M, N) \ + CK_CC_INLINE static T * \ + N(F *p) \ + { \ + F *n = p; \ + return (T *)(void *)(((char *)n) - ((size_t)&((T *)0)->M)); \ + } + +#define CK_CC_PAD(x) union { char pad[x]; } + +#ifndef CK_CC_ALIASED +#define CK_CC_ALIASED +#endif + +#ifndef CK_CC_UNUSED +#define CK_CC_UNUSED +#endif + +#ifndef CK_CC_USED +#define CK_CC_USED +#endif + +#ifndef CK_CC_IMM +#define CK_CC_IMM +#endif + +#ifndef CK_CC_PACKED +#define CK_CC_PACKED +#endif + +#ifndef CK_CC_WEAKREF +#define CK_CC_WEAKREF +#endif + +#ifndef CK_CC_ALIGN +#define CK_CC_ALIGN(X) +#endif + +#ifndef CK_CC_CACHELINE +#define CK_CC_CACHELINE +#endif + +#ifndef CK_CC_LIKELY +#define CK_CC_LIKELY(x) x +#endif + +#ifndef CK_CC_UNLIKELY +#define CK_CC_UNLIKELY(x) x +#endif + +#ifndef CK_CC_TYPEOF +#define CK_CC_TYPEOF(X, DEFAULT) (DEFAULT) +#endif + +#ifndef CK_F_CC_FFS +#define CK_F_CC_FFS +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 1; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif + +#ifndef CK_F_CC_CLZ +#define CK_F_CC_CLZ +#include <ck_limits.h> + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + unsigned int count, i; + + for (count = 0, i = sizeof(unsigned int) * CHAR_BIT; i > 0; count++) { + unsigned int bit = 1U << --i; + + if (x & bit) + break; + } + + return count; +} +#endif + +#ifndef CK_F_CC_CTZ +#define CK_F_CC_CTZ +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 0; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif + +#ifndef CK_F_CC_POPCOUNT +#define CK_F_CC_POPCOUNT +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + unsigned int acc; + + for (acc = 0; x != 0; x >>= 1) + acc += x & 1; + + return acc; +} +#endif + + +#ifdef __cplusplus +#define CK_CPP_CAST(type, arg) static_cast<type>(arg) +#else +#define CK_CPP_CAST(type, arg) arg +#endif + +#endif /* CK_CC_H */ diff --git a/include/ck_cohort.h b/include/ck_cohort.h new file mode 100644 index 0000000..c13cfa0 --- /dev/null +++ b/include/ck_cohort.h @@ -0,0 +1,161 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * Copyright 2013 Brendon Scheinman. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_COHORT_H +#define CK_COHORT_H + +/* + * This is an implementation of lock cohorts as described in: + * Dice, D.; Marathe, V.; and Shavit, N. 2012. + * Lock Cohorting: A General Technique for Designing NUMA Locks + */ + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stddef.h> + +enum ck_cohort_state { + CK_COHORT_STATE_GLOBAL = 0, + CK_COHORT_STATE_LOCAL = 1 +}; + +#define CK_COHORT_DEFAULT_LOCAL_PASS_LIMIT 10 + +#define CK_COHORT_NAME(N) ck_cohort_##N +#define CK_COHORT_INSTANCE(N) struct CK_COHORT_NAME(N) +#define CK_COHORT_INIT(N, C, GL, LL, P) ck_cohort_##N##_init(C, GL, LL, P) +#define CK_COHORT_LOCK(N, C, GC, LC) ck_cohort_##N##_lock(C, GC, LC) +#define CK_COHORT_UNLOCK(N, C, GC, LC) ck_cohort_##N##_unlock(C, GC, LC) +#define CK_COHORT_TRYLOCK(N, C, GLC, LLC, LUC) ck_cohort_##N##_trylock(C, GLC, LLC, LUC) +#define CK_COHORT_LOCKED(N, C, GC, LC) ck_cohort_##N##_locked(C, GC, LC) + +#define CK_COHORT_PROTOTYPE(N, GL, GU, GI, LL, LU, LI) \ + CK_COHORT_INSTANCE(N) { \ + void *global_lock; \ + void *local_lock; \ + enum ck_cohort_state release_state; \ + unsigned int waiting_threads; \ + unsigned int acquire_count; \ + unsigned int local_pass_limit; \ + }; \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_init(struct ck_cohort_##N *cohort, \ + void *global_lock, void *local_lock, unsigned int pass_limit) \ + { \ + cohort->global_lock = global_lock; \ + cohort->local_lock = local_lock; \ + cohort->release_state = CK_COHORT_STATE_GLOBAL; \ + cohort->waiting_threads = 0; \ + cohort->acquire_count = 0; \ + cohort->local_pass_limit = pass_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_lock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + \ + ck_pr_inc_uint(&cohort->waiting_threads); \ + LL(cohort->local_lock, local_context); \ + ck_pr_dec_uint(&cohort->waiting_threads); \ + \ + if (cohort->release_state == CK_COHORT_STATE_GLOBAL) { \ + GL(cohort->global_lock, global_context); \ + } \ + \ + ++cohort->acquire_count; \ + return; \ + } \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_unlock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + \ + if (ck_pr_load_uint(&cohort->waiting_threads) > 0 \ + && cohort->acquire_count < cohort->local_pass_limit) { \ + cohort->release_state = CK_COHORT_STATE_LOCAL; \ + } else { \ + GU(cohort->global_lock, global_context); \ + cohort->release_state = CK_COHORT_STATE_GLOBAL; \ + cohort->acquire_count = 0; \ + } \ + \ + ck_pr_fence_release(); \ + LU(cohort->local_lock, local_context); \ + \ + return; \ + } \ + \ + CK_CC_INLINE static bool \ + ck_cohort_##N##_locked(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + return GI(cohort->local_lock, local_context) || \ + LI(cohort->global_lock, global_context); \ + } + +#define CK_COHORT_TRYLOCK_PROTOTYPE(N, GL, GU, GI, GTL, LL, LU, LI, LTL) \ + CK_COHORT_PROTOTYPE(N, GL, GU, GI, LL, LU, LI) \ + CK_CC_INLINE static bool \ + ck_cohort_##N##_trylock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context, \ + void *local_unlock_context) \ + { \ + \ + bool trylock_result; \ + \ + ck_pr_inc_uint(&cohort->waiting_threads); \ + trylock_result = LTL(cohort->local_lock, local_context); \ + ck_pr_dec_uint(&cohort->waiting_threads); \ + if (trylock_result == false) { \ + return false; \ + } \ + \ + if (cohort->release_state == CK_COHORT_STATE_GLOBAL && \ + GTL(cohort->global_lock, global_context) == false) { \ + LU(cohort->local_lock, local_unlock_context); \ + return false; \ + } \ + \ + ++cohort->acquire_count; \ + return true; \ + } + +#define CK_COHORT_INITIALIZER { \ + .global_lock = NULL, \ + .local_lock = NULL, \ + .release_state = CK_COHORT_STATE_GLOBAL, \ + .waiting_threads = 0, \ + .acquire_count = 0, \ + .local_pass_limit = CK_COHORT_DEFAULT_LOCAL_PASS_LIMIT \ +} + +#endif /* CK_COHORT_H */ diff --git a/include/ck_elide.h b/include/ck_elide.h new file mode 100644 index 0000000..1b90041 --- /dev/null +++ b/include/ck_elide.h @@ -0,0 +1,321 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_ELIDE_H +#define CK_ELIDE_H + +/* + * As RTM is currently only supported on TSO x86 architectures, + * fences have been omitted. They will be necessary for other + * non-TSO architectures with TM support. + */ + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_string.h> + +/* + * skip_-prefixed counters represent the number of consecutive + * elisions to forfeit. retry_-prefixed counters represent the + * number of elision retries to attempt before forfeit. + * + * _busy: Lock was busy + * _other: Unknown explicit abort + * _conflict: Data conflict in elision section + */ +struct ck_elide_config { + unsigned short skip_busy; + short retry_busy; + unsigned short skip_other; + short retry_other; + unsigned short skip_conflict; + short retry_conflict; +}; + +#define CK_ELIDE_CONFIG_DEFAULT_INITIALIZER { \ + .skip_busy = 5, \ + .retry_busy = 256, \ + .skip_other = 3, \ + .retry_other = 3, \ + .skip_conflict = 2, \ + .retry_conflict = 5 \ +} + +struct ck_elide_stat { + unsigned int n_fallback; + unsigned int n_elide; + unsigned short skip; +}; +typedef struct ck_elide_stat ck_elide_stat_t; + +#define CK_ELIDE_STAT_INITIALIZER { 0, 0, 0 } + +CK_CC_INLINE static void +ck_elide_stat_init(ck_elide_stat_t *st) +{ + + memset(st, 0, sizeof(*st)); + return; +} + +#ifdef CK_F_PR_RTM +enum _ck_elide_hint { + CK_ELIDE_HINT_RETRY = 0, + CK_ELIDE_HINT_SPIN, + CK_ELIDE_HINT_STOP +}; + +#define CK_ELIDE_LOCK_BUSY 0xFF + +static enum _ck_elide_hint +_ck_elide_fallback(int *retry, + struct ck_elide_stat *st, + struct ck_elide_config *c, + unsigned int status) +{ + + st->n_fallback++; + if (*retry > 0) + return CK_ELIDE_HINT_RETRY; + + if (st->skip != 0) + return CK_ELIDE_HINT_STOP; + + if (status & CK_PR_RTM_EXPLICIT) { + if (CK_PR_RTM_CODE(status) == CK_ELIDE_LOCK_BUSY) { + st->skip = c->skip_busy; + *retry = c->retry_busy; + return CK_ELIDE_HINT_SPIN; + } + + st->skip = c->skip_other; + return CK_ELIDE_HINT_STOP; + } + + if ((status & CK_PR_RTM_RETRY) && + (status & CK_PR_RTM_CONFLICT)) { + st->skip = c->skip_conflict; + *retry = c->retry_conflict; + return CK_ELIDE_HINT_RETRY; + } + + /* + * Capacity, debug and nesting abortions are likely to be + * invariant conditions for the acquisition, execute regular + * path instead. If retry bit is not set, then take the hint. + */ + st->skip = USHRT_MAX; + return CK_ELIDE_HINT_STOP; +} + +/* + * Defines an elision implementation according to the following variables: + * N - Namespace of elision implementation. + * T - Typename of mutex. + * L_P - Lock predicate, returns false if resource is available. + * L - Function to call if resource is unavailable of transaction aborts. + * U_P - Unlock predicate, returns false if elision failed. + * U - Function to call if transaction failed. + */ +#define CK_ELIDE_PROTOTYPE(N, T, L_P, L, U_P, U) \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock_adaptive(T *lock, \ + struct ck_elide_stat *st, \ + struct ck_elide_config *c) \ + { \ + enum _ck_elide_hint hint; \ + int retry; \ + \ + if (CK_CC_UNLIKELY(st->skip != 0)) { \ + st->skip--; \ + goto acquire; \ + } \ + \ + retry = c->retry_conflict; \ + do { \ + unsigned int status = ck_pr_rtm_begin(); \ + if (status == CK_PR_RTM_STARTED) { \ + if (L_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return; \ + } \ + \ + hint = _ck_elide_fallback(&retry, st, c, status); \ + if (hint == CK_ELIDE_HINT_RETRY) \ + continue; \ + \ + if (hint == CK_ELIDE_HINT_SPIN) { \ + while (--retry != 0) { \ + if (L_P(lock) == false) \ + break; \ + \ + ck_pr_stall(); \ + } \ + \ + continue; \ + } \ + \ + if (hint == CK_ELIDE_HINT_STOP) \ + break; \ + } while (CK_CC_LIKELY(--retry > 0)); \ + \ + acquire: \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock_adaptive(struct ck_elide_stat *st, T *lock) \ + { \ + \ + if (U_P(lock) == false) { \ + ck_pr_rtm_end(); \ + st->skip = 0; \ + st->n_elide++; \ + } else { \ + U(lock); \ + } \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock(T *lock) \ + { \ + \ + if (ck_pr_rtm_begin() != CK_PR_RTM_STARTED) { \ + L(lock); \ + return; \ + } \ + \ + if (L_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock(T *lock) \ + { \ + \ + if (U_P(lock) == false) { \ + ck_pr_rtm_end(); \ + } else { \ + U(lock); \ + } \ + \ + return; \ + } + +#define CK_ELIDE_TRYLOCK_PROTOTYPE(N, T, TL_P, TL) \ + CK_CC_INLINE static bool \ + ck_elide_##N##_trylock(T *lock) \ + { \ + \ + if (ck_pr_rtm_begin() != CK_PR_RTM_STARTED) \ + return false; \ + \ + if (TL_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return true; \ + } +#else +/* + * If RTM is not enabled on the target platform (CK_F_PR_RTM) then these + * elision wrappers directly calls into the user-specified lock operations. + * Unfortunately, the storage cost of both ck_elide_config and ck_elide_stat + * are paid (typically a storage cost that is a function of lock objects and + * thread count). + */ +#define CK_ELIDE_PROTOTYPE(N, T, L_P, L, U_P, U) \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock_adaptive(T *lock, \ + struct ck_elide_stat *st, \ + struct ck_elide_config *c) \ + { \ + \ + (void)st; \ + (void)c; \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock_adaptive(struct ck_elide_stat *st, \ + T *lock) \ + { \ + \ + (void)st; \ + U(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock(T *lock) \ + { \ + \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock(T *lock) \ + { \ + \ + U(lock); \ + return; \ + } + +#define CK_ELIDE_TRYLOCK_PROTOTYPE(N, T, TL_P, TL) \ + CK_CC_INLINE static bool \ + ck_elide_##N##_trylock(T *lock) \ + { \ + \ + return TL_P(lock); \ + } +#endif /* !CK_F_PR_RTM */ + +/* + * Best-effort elision lock operations. First argument is name (N) + * associated with implementation and the second is a pointer to + * the type specified above (T). + * + * Unlike the adaptive variant, this interface does not have any retry + * semantics. In environments where jitter is low, this may yield a tighter + * fast path. + */ +#define CK_ELIDE_LOCK(NAME, LOCK) ck_elide_##NAME##_lock(LOCK) +#define CK_ELIDE_UNLOCK(NAME, LOCK) ck_elide_##NAME##_unlock(LOCK) +#define CK_ELIDE_TRYLOCK(NAME, LOCK) ck_elide_##NAME##_trylock(LOCK) + +/* + * Adaptive elision lock operations. In addition to name and pointer + * to the lock, you must pass in a pointer to an initialized + * ck_elide_config structure along with a per-thread stat structure. + */ +#define CK_ELIDE_LOCK_ADAPTIVE(NAME, STAT, CONFIG, LOCK) \ + ck_elide_##NAME##_lock_adaptive(LOCK, STAT, CONFIG) + +#define CK_ELIDE_UNLOCK_ADAPTIVE(NAME, STAT, LOCK) \ + ck_elide_##NAME##_unlock_adaptive(STAT, LOCK) + +#endif /* CK_ELIDE_H */ diff --git a/include/ck_epoch.h b/include/ck_epoch.h new file mode 100644 index 0000000..e7ce5bc --- /dev/null +++ b/include/ck_epoch.h @@ -0,0 +1,207 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_EPOCH_H +#define CK_EPOCH_H + +/* + * The implementation here is inspired from the work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + */ + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stack.h> +#include <ck_stdbool.h> + +#ifndef CK_EPOCH_LENGTH +#define CK_EPOCH_LENGTH 4 +#endif + +/* + * This is used for sense detection with-respect to concurrent + * epoch sections. + */ +#define CK_EPOCH_SENSE (2) + +struct ck_epoch_entry; +typedef struct ck_epoch_entry ck_epoch_entry_t; +typedef void ck_epoch_cb_t(ck_epoch_entry_t *); + +/* + * This should be embedded into objects you wish to be the target of + * ck_epoch_cb_t functions (with ck_epoch_call). + */ +struct ck_epoch_entry { + ck_epoch_cb_t *function; + ck_stack_entry_t stack_entry; +}; + +/* + * A section object may be passed to every begin-end pair to allow for + * forward progress guarantees with-in prolonged active sections. + */ +struct ck_epoch_section { + unsigned int bucket; +}; +typedef struct ck_epoch_section ck_epoch_section_t; + +/* + * Return pointer to ck_epoch_entry container object. + */ +#define CK_EPOCH_CONTAINER(T, M, N) \ + CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N) + +struct ck_epoch_ref { + unsigned int epoch; + unsigned int count; +}; + +struct ck_epoch_record { + struct ck_epoch *global; + unsigned int state; + unsigned int epoch; + unsigned int active; + struct { + struct ck_epoch_ref bucket[CK_EPOCH_SENSE]; + } local CK_CC_CACHELINE; + unsigned int n_pending; + unsigned int n_peak; + unsigned long n_dispatch; + ck_stack_t pending[CK_EPOCH_LENGTH]; + ck_stack_entry_t record_next; +} CK_CC_CACHELINE; +typedef struct ck_epoch_record ck_epoch_record_t; + +struct ck_epoch { + unsigned int epoch; + char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; + ck_stack_t records; + unsigned int n_free; +}; +typedef struct ck_epoch ck_epoch_t; + +/* + * Internal functions. + */ +void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *); +void _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *); + +/* + * Marks the beginning of an epoch-protected section. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_begin(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + struct ck_epoch *epoch = record->global; + + /* + * Only observe new epoch if thread is not recursing into a read + * section. + */ + if (record->active == 0) { + unsigned int g_epoch; + + /* + * It is possible for loads to be re-ordered before the store + * is committed into the caller's epoch and active fields. + * For this reason, store to load serialization is necessary. + */ +#if defined(CK_MD_TSO) + ck_pr_fas_uint(&record->active, 1); + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&record->active, 1); + ck_pr_fence_memory(); +#endif + + /* + * This load is allowed to be re-ordered prior to setting + * active flag due to monotonic nature of the global epoch. + * However, stale values lead to measurable performance + * degradation in some torture tests so we disallow early load + * of global epoch. + */ + g_epoch = ck_pr_load_uint(&epoch->epoch); + ck_pr_store_uint(&record->epoch, g_epoch); + } else { + ck_pr_store_uint(&record->active, record->active + 1); + } + + if (section != NULL) + _ck_epoch_addref(record, section); + + return; +} + +/* + * Marks the end of an epoch-protected section. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + + ck_pr_fence_release(); + ck_pr_store_uint(&record->active, record->active - 1); + + if (section != NULL) + _ck_epoch_delref(record, section); + + return; +} + +/* + * Defers the execution of the function pointed to by the "cb" + * argument until an epoch counter loop. This allows for a + * non-blocking deferral. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_call(ck_epoch_record_t *record, + ck_epoch_entry_t *entry, + ck_epoch_cb_t *function) +{ + struct ck_epoch *epoch = record->global; + unsigned int e = ck_pr_load_uint(&epoch->epoch); + unsigned int offset = e & (CK_EPOCH_LENGTH - 1); + + record->n_pending++; + entry->function = function; + ck_stack_push_spnc(&record->pending[offset], &entry->stack_entry); + return; +} + +void ck_epoch_init(ck_epoch_t *); +ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *); +void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *); +void ck_epoch_unregister(ck_epoch_record_t *); +bool ck_epoch_poll(ck_epoch_record_t *); +void ck_epoch_synchronize(ck_epoch_record_t *); +void ck_epoch_barrier(ck_epoch_record_t *); +void ck_epoch_reclaim(ck_epoch_record_t *); + +#endif /* CK_EPOCH_H */ diff --git a/include/ck_fifo.h b/include/ck_fifo.h new file mode 100644 index 0000000..6d50070 --- /dev/null +++ b/include/ck_fifo.h @@ -0,0 +1,478 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_FIFO_H +#define CK_FIFO_H + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_spinlock.h> +#include <ck_stddef.h> + +#ifndef CK_F_FIFO_SPSC +#define CK_F_FIFO_SPSC +struct ck_fifo_spsc_entry { + void *value; + struct ck_fifo_spsc_entry *next; +}; +typedef struct ck_fifo_spsc_entry ck_fifo_spsc_entry_t; + +struct ck_fifo_spsc { + ck_spinlock_t m_head; + struct ck_fifo_spsc_entry *head; + char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_spsc_entry *) - sizeof(ck_spinlock_t)]; + ck_spinlock_t m_tail; + struct ck_fifo_spsc_entry *tail; + struct ck_fifo_spsc_entry *head_snapshot; + struct ck_fifo_spsc_entry *garbage; +}; +typedef struct ck_fifo_spsc ck_fifo_spsc_t; + +CK_CC_INLINE static bool +ck_fifo_spsc_enqueue_trylock(struct ck_fifo_spsc *fifo) +{ + + return ck_spinlock_trylock(&fifo->m_tail); +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue_lock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_lock(&fifo->m_tail); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue_unlock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_unlock(&fifo->m_tail); + return; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_dequeue_trylock(struct ck_fifo_spsc *fifo) +{ + + return ck_spinlock_trylock(&fifo->m_head); +} + +CK_CC_INLINE static void +ck_fifo_spsc_dequeue_lock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_lock(&fifo->m_head); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_dequeue_unlock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_unlock(&fifo->m_head); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub) +{ + + ck_spinlock_init(&fifo->m_head); + ck_spinlock_init(&fifo->m_tail); + + stub->next = NULL; + fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub; + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_deinit(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry **garbage) +{ + + *garbage = fifo->head; + fifo->head = fifo->tail = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo, + struct ck_fifo_spsc_entry *entry, + void *value) +{ + + entry->value = value; + entry->next = NULL; + + /* If stub->next is visible, guarantee that entry is consistent. */ + ck_pr_fence_store(); + ck_pr_store_ptr(&fifo->tail->next, entry); + fifo->tail = entry; + return; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value) +{ + struct ck_fifo_spsc_entry *entry; + + /* + * The head pointer is guaranteed to always point to a stub entry. + * If the stub entry does not point to an entry, then the queue is + * empty. + */ + entry = ck_pr_load_ptr(&fifo->head->next); + if (entry == NULL) + return false; + + /* If entry is visible, guarantee store to value is visible. */ + ck_pr_store_ptr_unsafe(value, entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr(&fifo->head, entry); + return true; +} + +/* + * Recycle a node. This technique for recycling nodes is based on + * Dmitriy Vyukov's work. + */ +CK_CC_INLINE static struct ck_fifo_spsc_entry * +ck_fifo_spsc_recycle(struct ck_fifo_spsc *fifo) +{ + struct ck_fifo_spsc_entry *garbage; + + if (fifo->head_snapshot == fifo->garbage) { + fifo->head_snapshot = ck_pr_load_ptr(&fifo->head); + if (fifo->head_snapshot == fifo->garbage) + return NULL; + } + + garbage = fifo->garbage; + fifo->garbage = garbage->next; + return garbage; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_isempty(struct ck_fifo_spsc *fifo) +{ + struct ck_fifo_spsc_entry *head = ck_pr_load_ptr(&fifo->head); + return ck_pr_load_ptr(&head->next) == NULL; +} + +#define CK_FIFO_SPSC_ISEMPTY(f) ((f)->head->next == NULL) +#define CK_FIFO_SPSC_FIRST(f) ((f)->head->next) +#define CK_FIFO_SPSC_NEXT(m) ((m)->next) +#define CK_FIFO_SPSC_SPARE(f) ((f)->head) +#define CK_FIFO_SPSC_FOREACH(fifo, entry) \ + for ((entry) = CK_FIFO_SPSC_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_FIFO_SPSC_NEXT(entry)) +#define CK_FIFO_SPSC_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_FIFO_SPSC_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_F_FIFO_SPSC */ + +#ifdef CK_F_PR_CAS_PTR_2 +#ifndef CK_F_FIFO_MPMC +#define CK_F_FIFO_MPMC +struct ck_fifo_mpmc_entry; +struct ck_fifo_mpmc_pointer { + struct ck_fifo_mpmc_entry *pointer; + char *generation CK_CC_PACKED; +} CK_CC_ALIGN(16); + +struct ck_fifo_mpmc_entry { + void *value; + struct ck_fifo_mpmc_pointer next; +}; +typedef struct ck_fifo_mpmc_entry ck_fifo_mpmc_entry_t; + +struct ck_fifo_mpmc { + struct ck_fifo_mpmc_pointer head; + char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_mpmc_pointer)]; + struct ck_fifo_mpmc_pointer tail; +}; +typedef struct ck_fifo_mpmc ck_fifo_mpmc_t; + +CK_CC_INLINE static void +ck_fifo_mpmc_init(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *stub) +{ + + stub->next.pointer = NULL; + stub->next.generation = NULL; + fifo->head.pointer = fifo->tail.pointer = stub; + fifo->head.generation = fifo->tail.generation = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_mpmc_deinit(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry **garbage) +{ + + *garbage = fifo->head.pointer; + fifo->head.pointer = fifo->tail.pointer = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo, + struct ck_fifo_mpmc_entry *entry, + void *value) +{ + struct ck_fifo_mpmc_pointer tail, next, update; + + /* + * Prepare the upcoming node and make sure to commit the updates + * before publishing. + */ + entry->value = value; + entry->next.pointer = NULL; + entry->next.generation = 0; + ck_pr_fence_store_atomic(); + + for (;;) { + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); + + if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) + continue; + + if (next.pointer != NULL) { + /* + * If the tail pointer has an entry following it then + * it needs to be forwarded to the next entry. This + * helps us guarantee we are always operating on the + * last entry. + */ + update.pointer = next.pointer; + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + } else { + /* + * Attempt to commit new entry to the end of the + * current tail. + */ + update.pointer = entry; + update.generation = next.generation + 1; + if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == true) + break; + } + } + + ck_pr_fence_atomic(); + + /* After a successful insert, forward the tail to the new entry. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_tryenqueue(struct ck_fifo_mpmc *fifo, + struct ck_fifo_mpmc_entry *entry, + void *value) +{ + struct ck_fifo_mpmc_pointer tail, next, update; + + entry->value = value; + entry->next.pointer = NULL; + entry->next.generation = 0; + + ck_pr_fence_store_atomic(); + + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); + + if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) + return false; + + if (next.pointer != NULL) { + /* + * If the tail pointer has an entry following it then + * it needs to be forwarded to the next entry. This + * helps us guarantee we are always operating on the + * last entry. + */ + update.pointer = next.pointer; + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return false; + } else { + /* + * Attempt to commit new entry to the end of the + * current tail. + */ + update.pointer = entry; + update.generation = next.generation + 1; + if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == false) + return false; + } + + ck_pr_fence_atomic(); + + /* After a successful insert, forward the tail to the new entry. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return true; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo, + void *value, + struct ck_fifo_mpmc_entry **garbage) +{ + struct ck_fifo_mpmc_pointer head, tail, next, update; + + for (;;) { + head.generation = ck_pr_load_ptr(&fifo->head.generation); + ck_pr_fence_load(); + head.pointer = ck_pr_load_ptr(&fifo->head.pointer); + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + + next.generation = ck_pr_load_ptr(&head.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); + + update.pointer = next.pointer; + if (head.pointer == tail.pointer) { + /* + * The head is guaranteed to always point at a stub + * entry. If the stub entry has no references then the + * queue is empty. + */ + if (next.pointer == NULL) + return false; + + /* Forward the tail pointer if necessary. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + } else { + /* + * It is possible for head snapshot to have been + * re-used. Avoid deferencing during enqueue + * re-use. + */ + if (next.pointer == NULL) + continue; + + /* Save value before commit. */ + *(void **)value = ck_pr_load_ptr(&next.pointer->value); + + /* Forward the head pointer to the next entry. */ + update.generation = head.generation + 1; + if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == true) + break; + } + } + + *garbage = head.pointer; + return true; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_trydequeue(struct ck_fifo_mpmc *fifo, + void *value, + struct ck_fifo_mpmc_entry **garbage) +{ + struct ck_fifo_mpmc_pointer head, tail, next, update; + + head.generation = ck_pr_load_ptr(&fifo->head.generation); + ck_pr_fence_load(); + head.pointer = ck_pr_load_ptr(&fifo->head.pointer); + + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + + next.generation = ck_pr_load_ptr(&head.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); + + update.pointer = next.pointer; + if (head.pointer == tail.pointer) { + /* + * The head is guaranteed to always point at a stub + * entry. If the stub entry has no references then the + * queue is empty. + */ + if (next.pointer == NULL) + return false; + + /* Forward the tail pointer if necessary. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return false; + } else { + /* + * It is possible for head snapshot to have been + * re-used. Avoid deferencing during enqueue. + */ + if (next.pointer == NULL) + return false; + + /* Save value before commit. */ + *(void **)value = ck_pr_load_ptr(&next.pointer->value); + + /* Forward the head pointer to the next entry. */ + update.generation = head.generation + 1; + if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == false) + return false; + } + + *garbage = head.pointer; + return true; +} + +#define CK_FIFO_MPMC_ISEMPTY(f) ((f)->head.pointer->next.pointer == NULL) +#define CK_FIFO_MPMC_FIRST(f) ((f)->head.pointer->next.pointer) +#define CK_FIFO_MPMC_NEXT(m) ((m)->next.pointer) +#define CK_FIFO_MPMC_FOREACH(fifo, entry) \ + for ((entry) = CK_FIFO_MPMC_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_FIFO_MPMC_NEXT(entry)) +#define CK_FIFO_MPMC_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_FIFO_MPMC_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next.pointer, 1); \ + (entry) = (T)) + +#endif /* CK_F_FIFO_MPMC */ +#endif /* CK_F_PR_CAS_PTR_2 */ + +#endif /* CK_FIFO_H */ diff --git a/include/ck_hp.h b/include/ck_hp.h new file mode 100644 index 0000000..c7d8cfb --- /dev/null +++ b/include/ck_hp.h @@ -0,0 +1,121 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_H +#define CK_HP_H + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stack.h> + +#ifndef CK_HP_CACHE +#define CK_HP_CACHE 512 +#endif + +struct ck_hp_hazard; +typedef void (*ck_hp_destructor_t)(void *); + +struct ck_hp { + ck_stack_t subscribers; + unsigned int n_subscribers; + unsigned int n_free; + unsigned int threshold; + unsigned int degree; + ck_hp_destructor_t destroy; +}; +typedef struct ck_hp ck_hp_t; + +struct ck_hp_hazard { + void *pointer; + void *data; + ck_stack_entry_t pending_entry; +}; +typedef struct ck_hp_hazard ck_hp_hazard_t; + +enum { + CK_HP_USED = 0, + CK_HP_FREE = 1 +}; + +struct ck_hp_record { + int state; + void **pointers; + void *cache[CK_HP_CACHE]; + struct ck_hp *global; + ck_stack_t pending; + unsigned int n_pending; + ck_stack_entry_t global_entry; + unsigned int n_peak; + uint64_t n_reclamations; +} CK_CC_CACHELINE; +typedef struct ck_hp_record ck_hp_record_t; + +CK_CC_INLINE static void +ck_hp_set(struct ck_hp_record *record, unsigned int i, void *pointer) +{ + + ck_pr_store_ptr(&record->pointers[i], pointer); + return; +} + +CK_CC_INLINE static void +ck_hp_set_fence(struct ck_hp_record *record, unsigned int i, void *pointer) +{ + +#ifdef CK_MD_TSO + ck_pr_fas_ptr(&record->pointers[i], pointer); +#else + ck_pr_store_ptr(&record->pointers[i], pointer); + ck_pr_fence_memory(); +#endif + + return; +} + +CK_CC_INLINE static void +ck_hp_clear(struct ck_hp_record *record) +{ + void **pointers = record->pointers; + unsigned int i; + + for (i = 0; i < record->global->degree; i++) + *pointers++ = NULL; + + return; +} + +void ck_hp_init(ck_hp_t *, unsigned int, unsigned int, ck_hp_destructor_t); +void ck_hp_set_threshold(ck_hp_t *, unsigned int); +void ck_hp_register(ck_hp_t *, ck_hp_record_t *, void **); +void ck_hp_unregister(ck_hp_record_t *); +ck_hp_record_t *ck_hp_recycle(ck_hp_t *); +void ck_hp_reclaim(ck_hp_record_t *); +void ck_hp_free(ck_hp_record_t *, ck_hp_hazard_t *, void *, void *); +void ck_hp_retire(ck_hp_record_t *, ck_hp_hazard_t *, void *, void *); +void ck_hp_purge(ck_hp_record_t *); + +#endif /* CK_HP_H */ diff --git a/include/ck_hp_fifo.h b/include/ck_hp_fifo.h new file mode 100644 index 0000000..fd78ae6 --- /dev/null +++ b/include/ck_hp_fifo.h @@ -0,0 +1,215 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_FIFO_H +#define CK_HP_FIFO_H + +#include <ck_cc.h> +#include <ck_hp.h> +#include <ck_pr.h> +#include <ck_stddef.h> + +#define CK_HP_FIFO_SLOTS_COUNT (2) +#define CK_HP_FIFO_SLOTS_SIZE (sizeof(void *) * CK_HP_FIFO_SLOTS_COUNT) + +/* + * Though it is possible to embed the data structure, measurements need + * to be made for the cost of this. If we were to embed the hazard pointer + * state into the data structure, this means every deferred reclamation + * will also include a cache invalidation when linking into the hazard pointer + * pending queue. This may lead to terrible cache line bouncing. + */ +struct ck_hp_fifo_entry { + void *value; + ck_hp_hazard_t hazard; + struct ck_hp_fifo_entry *next; +}; +typedef struct ck_hp_fifo_entry ck_hp_fifo_entry_t; + +struct ck_hp_fifo { + struct ck_hp_fifo_entry *head; + struct ck_hp_fifo_entry *tail; +}; +typedef struct ck_hp_fifo ck_hp_fifo_t; + +CK_CC_INLINE static void +ck_hp_fifo_init(struct ck_hp_fifo *fifo, struct ck_hp_fifo_entry *stub) +{ + + fifo->head = fifo->tail = stub; + stub->next = NULL; + return; +} + +CK_CC_INLINE static void +ck_hp_fifo_deinit(struct ck_hp_fifo *fifo, struct ck_hp_fifo_entry **stub) +{ + + *stub = fifo->head; + fifo->head = fifo->tail = NULL; + return; +} + +CK_CC_INLINE static void +ck_hp_fifo_enqueue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + struct ck_hp_fifo_entry *entry, + void *value) +{ + struct ck_hp_fifo_entry *tail, *next; + + entry->value = value; + entry->next = NULL; + ck_pr_fence_store_atomic(); + + for (;;) { + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, tail); + if (tail != ck_pr_load_ptr(&fifo->tail)) + continue; + + next = ck_pr_load_ptr(&tail->next); + if (next != NULL) { + ck_pr_cas_ptr(&fifo->tail, tail, next); + continue; + } else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == true) + break; + } + + ck_pr_fence_atomic(); + ck_pr_cas_ptr(&fifo->tail, tail, entry); + return; +} + +CK_CC_INLINE static bool +ck_hp_fifo_tryenqueue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + struct ck_hp_fifo_entry *entry, + void *value) +{ + struct ck_hp_fifo_entry *tail, *next; + + entry->value = value; + entry->next = NULL; + ck_pr_fence_store_atomic(); + + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, tail); + if (tail != ck_pr_load_ptr(&fifo->tail)) + return false; + + next = ck_pr_load_ptr(&tail->next); + if (next != NULL) { + ck_pr_cas_ptr(&fifo->tail, tail, next); + return false; + } else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == false) + return false; + + ck_pr_fence_atomic(); + ck_pr_cas_ptr(&fifo->tail, tail, entry); + return true; +} + +CK_CC_INLINE static struct ck_hp_fifo_entry * +ck_hp_fifo_dequeue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + void *value) +{ + struct ck_hp_fifo_entry *head, *tail, *next; + + for (;;) { + head = ck_pr_load_ptr(&fifo->head); + ck_pr_fence_load(); + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, head); + if (head != ck_pr_load_ptr(&fifo->head)) + continue; + + next = ck_pr_load_ptr(&head->next); + ck_hp_set_fence(record, 1, next); + if (head != ck_pr_load_ptr(&fifo->head)) + continue; + + if (head == tail) { + if (next == NULL) + return NULL; + + ck_pr_cas_ptr(&fifo->tail, tail, next); + continue; + } else if (ck_pr_cas_ptr(&fifo->head, head, next) == true) + break; + } + + ck_pr_store_ptr_unsafe(value, next->value); + return head; +} + +CK_CC_INLINE static struct ck_hp_fifo_entry * +ck_hp_fifo_trydequeue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + void *value) +{ + struct ck_hp_fifo_entry *head, *tail, *next; + + head = ck_pr_load_ptr(&fifo->head); + ck_pr_fence_load(); + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, head); + if (head != ck_pr_load_ptr(&fifo->head)) + return NULL; + + next = ck_pr_load_ptr(&head->next); + ck_hp_set_fence(record, 1, next); + if (head != ck_pr_load_ptr(&fifo->head)) + return NULL; + + if (head == tail) { + if (next == NULL) + return NULL; + + ck_pr_cas_ptr(&fifo->tail, tail, next); + return NULL; + } else if (ck_pr_cas_ptr(&fifo->head, head, next) == false) + return NULL; + + ck_pr_store_ptr_unsafe(value, next->value); + return head; +} + +#define CK_HP_FIFO_ISEMPTY(f) ((f)->head->next == NULL) +#define CK_HP_FIFO_FIRST(f) ((f)->head->next) +#define CK_HP_FIFO_NEXT(m) ((m)->next) +#define CK_HP_FIFO_FOREACH(fifo, entry) \ + for ((entry) = CK_HP_FIFO_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_HP_FIFO_NEXT(entry)) +#define CK_HP_FIFO_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_HP_FIFO_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_HP_FIFO_H */ diff --git a/include/ck_hp_stack.h b/include/ck_hp_stack.h new file mode 100644 index 0000000..fb5a1e3 --- /dev/null +++ b/include/ck_hp_stack.h @@ -0,0 +1,110 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_STACK_H +#define CK_HP_STACK_H + +#include <ck_cc.h> +#include <ck_hp.h> +#include <ck_pr.h> +#include <ck_stack.h> +#include <ck_stddef.h> + +#define CK_HP_STACK_SLOTS_COUNT 1 +#define CK_HP_STACK_SLOTS_SIZE sizeof(void *) + +CK_CC_INLINE static void +ck_hp_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + ck_stack_push_upmc(target, entry); + return; +} + +CK_CC_INLINE static bool +ck_hp_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + return ck_stack_trypush_upmc(target, entry); +} + +CK_CC_INLINE static struct ck_stack_entry * +ck_hp_stack_pop_mpmc(ck_hp_record_t *record, struct ck_stack *target) +{ + struct ck_stack_entry *entry, *update; + + do { + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return NULL; + + ck_hp_set_fence(record, 0, entry); + } while (entry != ck_pr_load_ptr(&target->head)); + + while (ck_pr_cas_ptr_value(&target->head, entry, entry->next, &entry) == false) { + if (entry == NULL) + return NULL; + + ck_hp_set_fence(record, 0, entry); + + update = ck_pr_load_ptr(&target->head); + while (entry != update) { + ck_hp_set_fence(record, 0, update); + entry = update; + update = ck_pr_load_ptr(&target->head); + if (update == NULL) + return NULL; + } + } + + return entry; +} + +CK_CC_INLINE static bool +ck_hp_stack_trypop_mpmc(ck_hp_record_t *record, struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return false; + + ck_hp_set_fence(record, 0, entry); + if (entry != ck_pr_load_ptr(&target->head)) + goto leave; + + if (ck_pr_cas_ptr_value(&target->head, entry, entry->next, &entry) == false) + goto leave; + + *r = entry; + return true; + +leave: + ck_hp_set(record, 0, NULL); + return false; +} + +#endif /* CK_HP_STACK_H */ diff --git a/include/ck_hs.h b/include/ck_hs.h new file mode 100644 index 0000000..b3eb046 --- /dev/null +++ b/include/ck_hs.h @@ -0,0 +1,134 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HS_H +#define CK_HS_H + +#include <ck_cc.h> +#include <ck_malloc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +/* + * Indicates a single-writer many-reader workload. Mutually + * exclusive with CK_HS_MODE_MPMC + */ +#define CK_HS_MODE_SPMC 1 + +/* + * Indicates that values to be stored are not pointers but + * values. Allows for full precision. Mutually exclusive + * with CK_HS_MODE_OBJECT. + */ +#define CK_HS_MODE_DIRECT 2 + +/* + * Indicates that the values to be stored are pointers. + * Allows for space optimizations in the presence of pointer + * packing. Mutually exclusive with CK_HS_MODE_DIRECT. + */ +#define CK_HS_MODE_OBJECT 8 + +/* + * Indicates a delete-heavy workload. This will reduce the + * need for garbage collection at the cost of approximately + * 12% to 20% increased memory usage. + */ +#define CK_HS_MODE_DELETE 16 + +/* Currently unsupported. */ +#define CK_HS_MODE_MPMC (void) + +/* + * Hash callback function. + */ +typedef unsigned long ck_hs_hash_cb_t(const void *, unsigned long); + +/* + * Returns pointer to object if objects are equivalent. + */ +typedef bool ck_hs_compare_cb_t(const void *, const void *); + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_HS_PP +#define CK_HS_KEY_MASK ((1U << ((sizeof(void *) * 8) - CK_MD_VMA_BITS)) - 1) +#endif + +struct ck_hs_map; +struct ck_hs { + struct ck_malloc *m; + struct ck_hs_map *map; + unsigned int mode; + unsigned long seed; + ck_hs_hash_cb_t *hf; + ck_hs_compare_cb_t *compare; +}; +typedef struct ck_hs ck_hs_t; + +struct ck_hs_stat { + unsigned long tombstones; + unsigned long n_entries; + unsigned int probe_maximum; +}; + +struct ck_hs_iterator { + void **cursor; + unsigned long offset; +}; +typedef struct ck_hs_iterator ck_hs_iterator_t; + +#define CK_HS_ITERATOR_INITIALIZER { NULL, 0 } + +/* Convenience wrapper to table hash function. */ +#define CK_HS_HASH(T, F, K) F((K), (T)->seed) + +typedef void *ck_hs_apply_fn_t(void *, void *); +bool ck_hs_apply(ck_hs_t *, unsigned long, const void *, ck_hs_apply_fn_t *, void *); +void ck_hs_iterator_init(ck_hs_iterator_t *); +bool ck_hs_next(ck_hs_t *, ck_hs_iterator_t *, void **); +bool ck_hs_move(ck_hs_t *, ck_hs_t *, ck_hs_hash_cb_t *, + ck_hs_compare_cb_t *, struct ck_malloc *); +bool ck_hs_init(ck_hs_t *, unsigned int, ck_hs_hash_cb_t *, + ck_hs_compare_cb_t *, struct ck_malloc *, unsigned long, unsigned long); +void ck_hs_destroy(ck_hs_t *); +void *ck_hs_get(ck_hs_t *, unsigned long, const void *); +bool ck_hs_put(ck_hs_t *, unsigned long, const void *); +bool ck_hs_put_unique(ck_hs_t *, unsigned long, const void *); +bool ck_hs_set(ck_hs_t *, unsigned long, const void *, void **); +bool ck_hs_fas(ck_hs_t *, unsigned long, const void *, void **); +void *ck_hs_remove(ck_hs_t *, unsigned long, const void *); +bool ck_hs_grow(ck_hs_t *, unsigned long); +bool ck_hs_rebuild(ck_hs_t *); +bool ck_hs_gc(ck_hs_t *, unsigned long, unsigned long); +unsigned long ck_hs_count(ck_hs_t *); +bool ck_hs_reset(ck_hs_t *); +bool ck_hs_reset_size(ck_hs_t *, unsigned long); +void ck_hs_stat(ck_hs_t *, struct ck_hs_stat *); + +#endif /* CK_HS_H */ diff --git a/include/ck_ht.h b/include/ck_ht.h new file mode 100644 index 0000000..a949d30 --- /dev/null +++ b/include/ck_ht.h @@ -0,0 +1,271 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HT_H +#define CK_HT_H + +#include <ck_pr.h> + +#define CK_F_HT +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) +#define CK_HT_TYPE uint64_t +#define CK_HT_TYPE_LOAD ck_pr_load_64 +#define CK_HT_TYPE_STORE ck_pr_store_64 +#define CK_HT_TYPE_MAX UINT64_MAX +#else +#define CK_HT_TYPE uint32_t +#define CK_HT_TYPE_LOAD ck_pr_load_32 +#define CK_HT_TYPE_STORE ck_pr_store_32 +#define CK_HT_TYPE_MAX UINT32_MAX +#endif + + +#include <ck_cc.h> +#include <ck_malloc.h> +#include <ck_md.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_ht_hash { + uint64_t value; +}; +typedef struct ck_ht_hash ck_ht_hash_t; + +#define CK_HT_MODE_DIRECT 1U +#define CK_HT_MODE_BYTESTRING 2U +#define CK_HT_WORKLOAD_DELETE 4U + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_HT_PP +#define CK_HT_KEY_LENGTH ((sizeof(void *) * 8) - CK_MD_VMA_BITS) +#define CK_HT_KEY_MASK ((1U << CK_HT_KEY_LENGTH) - 1) +#else +#define CK_HT_KEY_LENGTH 65535U +#endif + +struct ck_ht_entry { +#ifdef CK_HT_PP + uintptr_t key; + uintptr_t value CK_CC_PACKED; +} CK_CC_ALIGN(16); +#else + uintptr_t key; + uintptr_t value; + CK_HT_TYPE key_length; + CK_HT_TYPE hash; +} CK_CC_ALIGN(32); +#endif +typedef struct ck_ht_entry ck_ht_entry_t; + +/* + * The user is free to define their own stub values. + */ +#ifndef CK_HT_KEY_EMPTY +#define CK_HT_KEY_EMPTY ((uintptr_t)0) +#endif + +#ifndef CK_HT_KEY_TOMBSTONE +#define CK_HT_KEY_TOMBSTONE (~CK_HT_KEY_EMPTY) +#endif + +/* + * Hash callback function. First argument is updated to contain a hash value, + * second argument is the key, third argument is key length and final argument + * is the hash table seed value. + */ +typedef void ck_ht_hash_cb_t(ck_ht_hash_t *, const void *, size_t, uint64_t); + +struct ck_ht_map; +struct ck_ht { + struct ck_malloc *m; + struct ck_ht_map *map; + unsigned int mode; + uint64_t seed; + ck_ht_hash_cb_t *h; +}; +typedef struct ck_ht ck_ht_t; + +struct ck_ht_stat { + uint64_t probe_maximum; + uint64_t n_entries; +}; + +struct ck_ht_iterator { + struct ck_ht_entry *current; + uint64_t offset; +}; +typedef struct ck_ht_iterator ck_ht_iterator_t; + +#define CK_HT_ITERATOR_INITIALIZER { NULL, 0 } + +CK_CC_INLINE static void +ck_ht_iterator_init(struct ck_ht_iterator *iterator) +{ + + iterator->current = NULL; + iterator->offset = 0; + return; +} + +CK_CC_INLINE static bool +ck_ht_entry_empty(ck_ht_entry_t *entry) +{ + + return entry->key == CK_HT_KEY_EMPTY; +} + +CK_CC_INLINE static void +ck_ht_entry_key_set_direct(ck_ht_entry_t *entry, uintptr_t key) +{ + + entry->key = key; + return; +} + +CK_CC_INLINE static void +ck_ht_entry_key_set(ck_ht_entry_t *entry, const void *key, uint16_t key_length) +{ + +#ifdef CK_HT_PP + entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS); +#else + entry->key = (uintptr_t)key; + entry->key_length = key_length; +#endif + + return; +} + +CK_CC_INLINE static void * +ck_ht_entry_key(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return (void *)(entry->key & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); +#else + return (void *)entry->key; +#endif +} + +CK_CC_INLINE static uint16_t +ck_ht_entry_key_length(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return entry->key >> CK_MD_VMA_BITS; +#else + return entry->key_length; +#endif +} + +CK_CC_INLINE static void * +ck_ht_entry_value(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return (void *)(entry->value & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); +#else + return (void *)entry->value; +#endif +} + +CK_CC_INLINE static void +ck_ht_entry_set(struct ck_ht_entry *entry, + ck_ht_hash_t h, + const void *key, + uint16_t key_length, + const void *value) +{ + +#ifdef CK_HT_PP + entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS); + entry->value = (uintptr_t)value | ((uintptr_t)(h.value >> 32) << CK_MD_VMA_BITS); +#else + entry->key = (uintptr_t)key; + entry->value = (uintptr_t)value; + entry->key_length = key_length; + entry->hash = h.value; +#endif + + return; +} + +CK_CC_INLINE static void +ck_ht_entry_set_direct(struct ck_ht_entry *entry, + ck_ht_hash_t h, + uintptr_t key, + uintptr_t value) +{ + + entry->key = key; + entry->value = value; + +#ifndef CK_HT_PP + entry->hash = h.value; +#else + (void)h; +#endif + return; +} + +CK_CC_INLINE static uintptr_t +ck_ht_entry_key_direct(ck_ht_entry_t *entry) +{ + + return entry->key; +} + +CK_CC_INLINE static uintptr_t +ck_ht_entry_value_direct(ck_ht_entry_t *entry) +{ + + return entry->value; +} + +/* + * Iteration must occur without any concurrent mutations on + * the hash table. + */ +bool ck_ht_next(ck_ht_t *, ck_ht_iterator_t *, ck_ht_entry_t **entry); + +void ck_ht_stat(ck_ht_t *, struct ck_ht_stat *); +void ck_ht_hash(ck_ht_hash_t *, ck_ht_t *, const void *, uint16_t); +void ck_ht_hash_direct(ck_ht_hash_t *, ck_ht_t *, uintptr_t); +bool ck_ht_init(ck_ht_t *, unsigned int, ck_ht_hash_cb_t *, + struct ck_malloc *, CK_HT_TYPE, uint64_t); +void ck_ht_destroy(ck_ht_t *); +bool ck_ht_set_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_put_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_get_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_gc(struct ck_ht *, unsigned long, unsigned long); +bool ck_ht_grow_spmc(ck_ht_t *, CK_HT_TYPE); +bool ck_ht_remove_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_reset_spmc(ck_ht_t *); +bool ck_ht_reset_size_spmc(ck_ht_t *, CK_HT_TYPE); +CK_HT_TYPE ck_ht_count(ck_ht_t *); + +#endif /* CK_HT_H */ diff --git a/include/ck_limits.h b/include/ck_limits.h new file mode 100644 index 0000000..c874955 --- /dev/null +++ b/include/ck_limits.h @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include <linux/kernel.h> + +#ifndef UINT8_MAX +#define UINT8_MAX ((u8)(~0U)) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX USHRT_MAX +#endif +#ifndef UINT32_MAX +#define UINT32_MAX UINT_MAX +#endif +#ifndef UINT64_MAX +#define UINT64_MAX ULLONG_MAX +#endif + +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stdint.h> +#include <sys/limits.h> +#else +#include <limits.h> +#endif /* __linux__ && __KERNEL__ */ diff --git a/include/ck_malloc.h b/include/ck_malloc.h new file mode 100644 index 0000000..e14dde3 --- /dev/null +++ b/include/ck_malloc.h @@ -0,0 +1,39 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_MALLOC_H +#define CK_MALLOC_H + +#include <ck_stdbool.h> +#include <sys/types.h> + +struct ck_malloc { + void *(*malloc)(size_t); + void *(*realloc)(void *, size_t, size_t, bool); + void (*free)(void *, size_t, bool); +}; + +#endif /* CK_MALLOC_H */ diff --git a/include/ck_md.h.in b/include/ck_md.h.in new file mode 100644 index 0000000..cb5783e --- /dev/null +++ b/include/ck_md.h.in @@ -0,0 +1,65 @@ +/* + * Copyright 2011-2012 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_MD_H +#define CK_MD_H + +#ifndef CK_MD_CACHELINE +#define CK_MD_CACHELINE (64) +#endif + +#ifndef CK_MD_PAGESIZE +#define CK_MD_PAGESIZE (4096) +#endif + +#ifndef @RTM_ENABLE@ +#define @RTM_ENABLE@ +#endif /* @RTM_ENABLE@ */ + +#ifndef @LSE_ENABLE@ +#define @LSE_ENABLE@ +#endif /* @LSE_ENABLE@ */ + +#ifndef @POINTER_PACK_ENABLE@ +#define @POINTER_PACK_ENABLE@ +#endif /* @POINTER_PACK_ENABLE@ */ + +#ifndef @VMA_BITS@ +#define @VMA_BITS@ @VMA_BITS_VALUE@ +#endif /* @VMA_BITS@ */ + +#ifndef @MM@ +#define @MM@ +#endif /* @MM@ */ + +#ifndef @DISABLE_DOUBLE@ +#define @DISABLE_DOUBLE@ +#endif /* @DISABLE_DOUBLE@ */ + +#define CK_VERSION "@VERSION@" +#define CK_GIT_SHA "@GIT_SHA@" + +#endif /* CK_MD_H */ diff --git a/include/ck_pflock.h b/include/ck_pflock.h new file mode 100644 index 0000000..61b42bd --- /dev/null +++ b/include/ck_pflock.h @@ -0,0 +1,142 @@ +/* + * Copyright 2013 John Wittrock. + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PFLOCK_H +#define CK_PFLOCK_H + +/* + * This is an implementation of phase-fair locks derived from the work + * described in: + * Brandenburg, B. and Anderson, J. 2010. Spin-Based + * Reader-Writer Synchronization for Multiprocessor Real-Time Systems + */ + +#include <ck_cc.h> +#include <ck_pr.h> + +struct ck_pflock { + uint32_t rin; + uint32_t rout; + uint32_t win; + uint32_t wout; +}; +typedef struct ck_pflock ck_pflock_t; + +#define CK_PFLOCK_LSB 0xFFFFFFF0 +#define CK_PFLOCK_RINC 0x100 /* Reader increment value. */ +#define CK_PFLOCK_WBITS 0x3 /* Writer bits in reader. */ +#define CK_PFLOCK_PRES 0x2 /* Writer present bit. */ +#define CK_PFLOCK_PHID 0x1 /* Phase ID bit. */ + +#define CK_PFLOCK_INITIALIZER {0, 0, 0, 0} + +CK_CC_INLINE static void +ck_pflock_init(struct ck_pflock *pf) +{ + + pf->rin = 0; + pf->rout = 0; + pf->win = 0; + pf->wout = 0; + ck_pr_barrier(); + + return; +} + +CK_CC_INLINE static void +ck_pflock_write_unlock(ck_pflock_t *pf) +{ + + ck_pr_fence_unlock(); + + /* Migrate from write phase to read phase. */ + ck_pr_and_32(&pf->rin, CK_PFLOCK_LSB); + + /* Allow other writers to continue. */ + ck_pr_faa_32(&pf->wout, 1); + return; +} + +CK_CC_INLINE static void +ck_pflock_write_lock(ck_pflock_t *pf) +{ + uint32_t ticket; + + /* Acquire ownership of write-phase. */ + ticket = ck_pr_faa_32(&pf->win, 1); + while (ck_pr_load_32(&pf->wout) != ticket) + ck_pr_stall(); + + /* + * Acquire ticket on read-side in order to allow them + * to flush. Indicates to any incoming reader that a + * write-phase is pending. + */ + ticket = ck_pr_faa_32(&pf->rin, + (ticket & CK_PFLOCK_PHID) | CK_PFLOCK_PRES); + + /* Wait for any pending readers to flush. */ + while (ck_pr_load_32(&pf->rout) != ticket) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_pflock_read_unlock(ck_pflock_t *pf) +{ + + ck_pr_fence_unlock(); + ck_pr_faa_32(&pf->rout, CK_PFLOCK_RINC); + return; +} + +CK_CC_INLINE static void +ck_pflock_read_lock(ck_pflock_t *pf) +{ + uint32_t w; + + /* + * If no writer is present, then the operation has completed + * successfully. + */ + w = ck_pr_faa_32(&pf->rin, CK_PFLOCK_RINC) & CK_PFLOCK_WBITS; + if (w == 0) + goto leave; + + /* Wait for current write phase to complete. */ + while ((ck_pr_load_32(&pf->rin) & CK_PFLOCK_WBITS) == w) + ck_pr_stall(); + +leave: + /* Acquire semantics with respect to readers. */ + ck_pr_fence_lock(); + return; +} + +#endif /* CK_PFLOCK_H */ diff --git a/include/ck_pr.h b/include/ck_pr.h new file mode 100644 index 0000000..9b7fc42 --- /dev/null +++ b/include/ck_pr.h @@ -0,0 +1,1219 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_H +#define CK_PR_H + +#include <ck_cc.h> +#include <ck_limits.h> +#include <ck_md.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> + +#ifndef CK_USE_CC_BUILTINS +#if defined(__x86_64__) +#include "gcc/x86_64/ck_pr.h" +#elif defined(__x86__) +#include "gcc/x86/ck_pr.h" +#elif defined(__sparcv9__) +#include "gcc/sparcv9/ck_pr.h" +#elif defined(__ppc64__) +#include "gcc/ppc64/ck_pr.h" +#elif defined(__ppc__) +#include "gcc/ppc/ck_pr.h" +#elif defined(__arm__) +#include "gcc/arm/ck_pr.h" +#elif defined(__aarch64__) +#include "gcc/aarch64/ck_pr.h" +#elif !defined(__GNUC__) +#error Your platform is unsupported +#endif +#endif /* !CK_USE_CC_BUILTINS */ + +#if defined(__GNUC__) +#include "gcc/ck_pr.h" +#endif + +#define CK_PR_FENCE_EMIT(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_fence_strict_##T(); \ + return; \ + } +#define CK_PR_FENCE_NOOP(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_barrier(); \ + return; \ + } + +/* + * None of the currently supported platforms allow for data-dependent + * load ordering. + */ +CK_PR_FENCE_NOOP(load_depends) +#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends + +/* + * In memory models where atomic operations do not have serializing + * effects, atomic read-modify-write operations are modeled as stores. + */ +#if defined(CK_MD_RMO) +/* + * Only stores to the same location have a global + * ordering. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_EMIT(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_EMIT(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_EMIT(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_PSO) +/* + * Anything can be re-ordered with respect to stores. + * Otherwise, loads are executed in-order. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_TSO) +/* + * Only loads are re-ordered and only with respect to + * prior stores. Atomic operations are serializing. + */ +CK_PR_FENCE_NOOP(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_NOOP(atomic_store) +CK_PR_FENCE_NOOP(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_NOOP(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_NOOP(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_NOOP(acquire) +CK_PR_FENCE_NOOP(release) +CK_PR_FENCE_NOOP(acqrel) +CK_PR_FENCE_NOOP(lock) +CK_PR_FENCE_NOOP(unlock) +#else +#error "No memory model has been defined." +#endif /* CK_MD_TSO */ + +#undef CK_PR_FENCE_EMIT +#undef CK_PR_FENCE_NOOP + +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + (void)m; + return; +} +#endif /* CK_F_PR_RFO */ + +#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ + ck_pr_md_store_##TYPE( \ + ((void)sizeof(*(DST) = (VAL)), (DST)), \ + (VAL)) + +#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) +#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) +#endif +#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) +#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) +#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) +#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) +#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) + +#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) +#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) + +#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) +#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) +#endif +#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) +#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) +#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) +#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) +#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_BIN(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T value) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P value), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_CHAR +CK_PR_BIN_S(add, char, char, +) +#endif /* CK_F_PR_ADD_CHAR */ + +#ifndef CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_CHAR +CK_PR_BIN_S(sub, char, char, -) +#endif /* CK_F_PR_SUB_CHAR */ + +#ifndef CK_F_PR_AND_CHAR +#define CK_F_PR_AND_CHAR +CK_PR_BIN_S(and, char, char, &) +#endif /* CK_F_PR_AND_CHAR */ + +#ifndef CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_CHAR +CK_PR_BIN_S(xor, char, char, ^) +#endif /* CK_F_PR_XOR_CHAR */ + +#ifndef CK_F_PR_OR_CHAR +#define CK_F_PR_OR_CHAR +CK_PR_BIN_S(or, char, char, |) +#endif /* CK_F_PR_OR_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_ADD_INT +#define CK_F_PR_ADD_INT +CK_PR_BIN_S(add, int, int, +) +#endif /* CK_F_PR_ADD_INT */ + +#ifndef CK_F_PR_SUB_INT +#define CK_F_PR_SUB_INT +CK_PR_BIN_S(sub, int, int, -) +#endif /* CK_F_PR_SUB_INT */ + +#ifndef CK_F_PR_AND_INT +#define CK_F_PR_AND_INT +CK_PR_BIN_S(and, int, int, &) +#endif /* CK_F_PR_AND_INT */ + +#ifndef CK_F_PR_XOR_INT +#define CK_F_PR_XOR_INT +CK_PR_BIN_S(xor, int, int, ^) +#endif /* CK_F_PR_XOR_INT */ + +#ifndef CK_F_PR_OR_INT +#define CK_F_PR_OR_INT +CK_PR_BIN_S(or, int, int, |) +#endif /* CK_F_PR_OR_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_ADD_DOUBLE +#define CK_F_PR_ADD_DOUBLE +CK_PR_BIN_S(add, double, double, +) +#endif /* CK_F_PR_ADD_DOUBLE */ + +#ifndef CK_F_PR_SUB_DOUBLE +#define CK_F_PR_SUB_DOUBLE +CK_PR_BIN_S(sub, double, double, -) +#endif /* CK_F_PR_SUB_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_ADD_UINT +#define CK_F_PR_ADD_UINT +CK_PR_BIN_S(add, uint, unsigned int, +) +#endif /* CK_F_PR_ADD_UINT */ + +#ifndef CK_F_PR_SUB_UINT +#define CK_F_PR_SUB_UINT +CK_PR_BIN_S(sub, uint, unsigned int, -) +#endif /* CK_F_PR_SUB_UINT */ + +#ifndef CK_F_PR_AND_UINT +#define CK_F_PR_AND_UINT +CK_PR_BIN_S(and, uint, unsigned int, &) +#endif /* CK_F_PR_AND_UINT */ + +#ifndef CK_F_PR_XOR_UINT +#define CK_F_PR_XOR_UINT +CK_PR_BIN_S(xor, uint, unsigned int, ^) +#endif /* CK_F_PR_XOR_UINT */ + +#ifndef CK_F_PR_OR_UINT +#define CK_F_PR_OR_UINT +CK_PR_BIN_S(or, uint, unsigned int, |) +#endif /* CK_F_PR_OR_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_PTR +CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) +#endif /* CK_F_PR_ADD_PTR */ + +#ifndef CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_PTR +CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_SUB_PTR */ + +#ifndef CK_F_PR_AND_PTR +#define CK_F_PR_AND_PTR +CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) +#endif /* CK_F_PR_AND_PTR */ + +#ifndef CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_PTR +CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) +#endif /* CK_F_PR_XOR_PTR */ + +#ifndef CK_F_PR_OR_PTR +#define CK_F_PR_OR_PTR +CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) +#endif /* CK_F_PR_OR_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_ADD_64 +#define CK_F_PR_ADD_64 +CK_PR_BIN_S(add, 64, uint64_t, +) +#endif /* CK_F_PR_ADD_64 */ + +#ifndef CK_F_PR_SUB_64 +#define CK_F_PR_SUB_64 +CK_PR_BIN_S(sub, 64, uint64_t, -) +#endif /* CK_F_PR_SUB_64 */ + +#ifndef CK_F_PR_AND_64 +#define CK_F_PR_AND_64 +CK_PR_BIN_S(and, 64, uint64_t, &) +#endif /* CK_F_PR_AND_64 */ + +#ifndef CK_F_PR_XOR_64 +#define CK_F_PR_XOR_64 +CK_PR_BIN_S(xor, 64, uint64_t, ^) +#endif /* CK_F_PR_XOR_64 */ + +#ifndef CK_F_PR_OR_64 +#define CK_F_PR_OR_64 +CK_PR_BIN_S(or, 64, uint64_t, |) +#endif /* CK_F_PR_OR_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_ADD_32 +#define CK_F_PR_ADD_32 +CK_PR_BIN_S(add, 32, uint32_t, +) +#endif /* CK_F_PR_ADD_32 */ + +#ifndef CK_F_PR_SUB_32 +#define CK_F_PR_SUB_32 +CK_PR_BIN_S(sub, 32, uint32_t, -) +#endif /* CK_F_PR_SUB_32 */ + +#ifndef CK_F_PR_AND_32 +#define CK_F_PR_AND_32 +CK_PR_BIN_S(and, 32, uint32_t, &) +#endif /* CK_F_PR_AND_32 */ + +#ifndef CK_F_PR_XOR_32 +#define CK_F_PR_XOR_32 +CK_PR_BIN_S(xor, 32, uint32_t, ^) +#endif /* CK_F_PR_XOR_32 */ + +#ifndef CK_F_PR_OR_32 +#define CK_F_PR_OR_32 +CK_PR_BIN_S(or, 32, uint32_t, |) +#endif /* CK_F_PR_OR_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_ADD_16 +#define CK_F_PR_ADD_16 +CK_PR_BIN_S(add, 16, uint16_t, +) +#endif /* CK_F_PR_ADD_16 */ + +#ifndef CK_F_PR_SUB_16 +#define CK_F_PR_SUB_16 +CK_PR_BIN_S(sub, 16, uint16_t, -) +#endif /* CK_F_PR_SUB_16 */ + +#ifndef CK_F_PR_AND_16 +#define CK_F_PR_AND_16 +CK_PR_BIN_S(and, 16, uint16_t, &) +#endif /* CK_F_PR_AND_16 */ + +#ifndef CK_F_PR_XOR_16 +#define CK_F_PR_XOR_16 +CK_PR_BIN_S(xor, 16, uint16_t, ^) +#endif /* CK_F_PR_XOR_16 */ + +#ifndef CK_F_PR_OR_16 +#define CK_F_PR_OR_16 +CK_PR_BIN_S(or, 16, uint16_t, |) +#endif /* CK_F_PR_OR_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_ADD_8 +#define CK_F_PR_ADD_8 +CK_PR_BIN_S(add, 8, uint8_t, +) +#endif /* CK_F_PR_ADD_8 */ + +#ifndef CK_F_PR_SUB_8 +#define CK_F_PR_SUB_8 +CK_PR_BIN_S(sub, 8, uint8_t, -) +#endif /* CK_F_PR_SUB_8 */ + +#ifndef CK_F_PR_AND_8 +#define CK_F_PR_AND_8 +CK_PR_BIN_S(and, 8, uint8_t, &) +#endif /* CK_F_PR_AND_8 */ + +#ifndef CK_F_PR_XOR_8 +#define CK_F_PR_XOR_8 +CK_PR_BIN_S(xor, 8, uint8_t, ^) +#endif /* CK_F_PR_XOR_8 */ + +#ifndef CK_F_PR_OR_8 +#define CK_F_PR_OR_8 +CK_PR_BIN_S(or, 8, uint8_t, |) +#endif /* CK_F_PR_OR_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_BIN_S +#undef CK_PR_BIN + +#define CK_PR_BTX(K, S, M, T, P, C, R) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(M *target, unsigned int offset) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, (C)previous, \ + (C)(previous P (R ((T)1 << offset))), &previous) == false) \ + ck_pr_stall(); \ + return ((previous >> offset) & 1); \ + } + +#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_BTC_INT +#define CK_F_PR_BTC_INT +CK_PR_BTX_S(btc, int, int, ^,) +#endif /* CK_F_PR_BTC_INT */ + +#ifndef CK_F_PR_BTR_INT +#define CK_F_PR_BTR_INT +CK_PR_BTX_S(btr, int, int, &, ~) +#endif /* CK_F_PR_BTR_INT */ + +#ifndef CK_F_PR_BTS_INT +#define CK_F_PR_BTS_INT +CK_PR_BTX_S(bts, int, int, |,) +#endif /* CK_F_PR_BTS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_BTC_UINT +#define CK_F_PR_BTC_UINT +CK_PR_BTX_S(btc, uint, unsigned int, ^,) +#endif /* CK_F_PR_BTC_UINT */ + +#ifndef CK_F_PR_BTR_UINT +#define CK_F_PR_BTR_UINT +CK_PR_BTX_S(btr, uint, unsigned int, &, ~) +#endif /* CK_F_PR_BTR_UINT */ + +#ifndef CK_F_PR_BTS_UINT +#define CK_F_PR_BTS_UINT +CK_PR_BTX_S(bts, uint, unsigned int, |,) +#endif /* CK_F_PR_BTS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_PTR +CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) +#endif /* CK_F_PR_BTC_PTR */ + +#ifndef CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_PTR +CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) +#endif /* CK_F_PR_BTR_PTR */ + +#ifndef CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_PTR +CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) +#endif /* CK_F_PR_BTS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_BTC_64 +#define CK_F_PR_BTC_64 +CK_PR_BTX_S(btc, 64, uint64_t, ^,) +#endif /* CK_F_PR_BTC_64 */ + +#ifndef CK_F_PR_BTR_64 +#define CK_F_PR_BTR_64 +CK_PR_BTX_S(btr, 64, uint64_t, &, ~) +#endif /* CK_F_PR_BTR_64 */ + +#ifndef CK_F_PR_BTS_64 +#define CK_F_PR_BTS_64 +CK_PR_BTX_S(bts, 64, uint64_t, |,) +#endif /* CK_F_PR_BTS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_BTC_32 +#define CK_F_PR_BTC_32 +CK_PR_BTX_S(btc, 32, uint32_t, ^,) +#endif /* CK_F_PR_BTC_32 */ + +#ifndef CK_F_PR_BTR_32 +#define CK_F_PR_BTR_32 +CK_PR_BTX_S(btr, 32, uint32_t, &, ~) +#endif /* CK_F_PR_BTR_32 */ + +#ifndef CK_F_PR_BTS_32 +#define CK_F_PR_BTS_32 +CK_PR_BTX_S(bts, 32, uint32_t, |,) +#endif /* CK_F_PR_BTS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_BTC_16 +#define CK_F_PR_BTC_16 +CK_PR_BTX_S(btc, 16, uint16_t, ^,) +#endif /* CK_F_PR_BTC_16 */ + +#ifndef CK_F_PR_BTR_16 +#define CK_F_PR_BTR_16 +CK_PR_BTX_S(btr, 16, uint16_t, &, ~) +#endif /* CK_F_PR_BTR_16 */ + +#ifndef CK_F_PR_BTS_16 +#define CK_F_PR_BTS_16 +CK_PR_BTX_S(bts, 16, uint16_t, |,) +#endif /* CK_F_PR_BTS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#undef CK_PR_BTX_S +#undef CK_PR_BTX + +#define CK_PR_UNARY(K, X, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + ck_pr_##X##_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P 1), \ + &previous) == false) \ + ck_pr_stall(); \ + *zero = previous == (T)Z; \ + return; \ + } + +#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) +#define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR +CK_PR_UNARY_S(inc, add, char, char) +#endif /* CK_F_PR_INC_CHAR */ + +#ifndef CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_CHAR_ZERO +CK_PR_UNARY_Z_S(inc, char, char, +, -1) +#endif /* CK_F_PR_INC_CHAR_ZERO */ + +#ifndef CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR +CK_PR_UNARY_S(dec, sub, char, char) +#endif /* CK_F_PR_DEC_CHAR */ + +#ifndef CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_CHAR_ZERO +CK_PR_UNARY_Z_S(dec, char, char, -, 1) +#endif /* CK_F_PR_DEC_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_INC_INT +#define CK_F_PR_INC_INT +CK_PR_UNARY_S(inc, add, int, int) +#endif /* CK_F_PR_INC_INT */ + +#ifndef CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_INT_ZERO +CK_PR_UNARY_Z_S(inc, int, int, +, -1) +#endif /* CK_F_PR_INC_INT_ZERO */ + +#ifndef CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT +CK_PR_UNARY_S(dec, sub, int, int) +#endif /* CK_F_PR_DEC_INT */ + +#ifndef CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_INT_ZERO +CK_PR_UNARY_Z_S(dec, int, int, -, 1) +#endif /* CK_F_PR_DEC_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_INC_DOUBLE +#define CK_F_PR_INC_DOUBLE +CK_PR_UNARY_S(inc, add, double, double) +#endif /* CK_F_PR_INC_DOUBLE */ + +#ifndef CK_F_PR_DEC_DOUBLE +#define CK_F_PR_DEC_DOUBLE +CK_PR_UNARY_S(dec, sub, double, double) +#endif /* CK_F_PR_DEC_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT +CK_PR_UNARY_S(inc, add, uint, unsigned int) +#endif /* CK_F_PR_INC_UINT */ + +#ifndef CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_INC_UINT_ZERO +CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) +#endif /* CK_F_PR_INC_UINT_ZERO */ + +#ifndef CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT +CK_PR_UNARY_S(dec, sub, uint, unsigned int) +#endif /* CK_F_PR_DEC_UINT */ + +#ifndef CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_DEC_UINT_ZERO +CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) +#endif /* CK_F_PR_DEC_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR +CK_PR_UNARY(inc, add, ptr, void, uintptr_t) +#endif /* CK_F_PR_INC_PTR */ + +#ifndef CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_PTR_ZERO +CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) +#endif /* CK_F_PR_INC_PTR_ZERO */ + +#ifndef CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR +CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) +#endif /* CK_F_PR_DEC_PTR */ + +#ifndef CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_PTR_ZERO +CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) +#endif /* CK_F_PR_DEC_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_INC_64 +#define CK_F_PR_INC_64 +CK_PR_UNARY_S(inc, add, 64, uint64_t) +#endif /* CK_F_PR_INC_64 */ + +#ifndef CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_64_ZERO +CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) +#endif /* CK_F_PR_INC_64_ZERO */ + +#ifndef CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64 +CK_PR_UNARY_S(dec, sub, 64, uint64_t) +#endif /* CK_F_PR_DEC_64 */ + +#ifndef CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_64_ZERO +CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) +#endif /* CK_F_PR_DEC_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_INC_32 +#define CK_F_PR_INC_32 +CK_PR_UNARY_S(inc, add, 32, uint32_t) +#endif /* CK_F_PR_INC_32 */ + +#ifndef CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_32_ZERO +CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) +#endif /* CK_F_PR_INC_32_ZERO */ + +#ifndef CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32 +CK_PR_UNARY_S(dec, sub, 32, uint32_t) +#endif /* CK_F_PR_DEC_32 */ + +#ifndef CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_32_ZERO +CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) +#endif /* CK_F_PR_DEC_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_INC_16 +#define CK_F_PR_INC_16 +CK_PR_UNARY_S(inc, add, 16, uint16_t) +#endif /* CK_F_PR_INC_16 */ + +#ifndef CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_16_ZERO +CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) +#endif /* CK_F_PR_INC_16_ZERO */ + +#ifndef CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16 +CK_PR_UNARY_S(dec, sub, 16, uint16_t) +#endif /* CK_F_PR_DEC_16 */ + +#ifndef CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_16_ZERO +CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) +#endif /* CK_F_PR_DEC_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_INC_8 +#define CK_F_PR_INC_8 +CK_PR_UNARY_S(inc, add, 8, uint8_t) +#endif /* CK_F_PR_INC_8 */ + +#ifndef CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_8_ZERO +CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) +#endif /* CK_F_PR_INC_8_ZERO */ + +#ifndef CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8 +CK_PR_UNARY_S(dec, sub, 8, uint8_t) +#endif /* CK_F_PR_DEC_8 */ + +#ifndef CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_8_ZERO +CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) +#endif /* CK_F_PR_DEC_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_UNARY_Z_S +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_Z +#undef CK_PR_UNARY + +#define CK_PR_N(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(P previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_N_Z(S, M, T, C) \ + CK_CC_INLINE static void \ + ck_pr_neg_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(-previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + *zero = previous == 0; \ + return; \ + } + +#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) +#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_CHAR +CK_PR_N_S(not, char, char, ~) +#endif /* CK_F_PR_NOT_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR +CK_PR_N_S(neg, char, char, -) +#endif /* CK_F_PR_NEG_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_CHAR_ZERO +CK_PR_N_Z_S(char, char) +#endif /* CK_F_PR_NEG_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_NOT_INT +#define CK_F_PR_NOT_INT +CK_PR_N_S(not, int, int, ~) +#endif /* CK_F_PR_NOT_INT */ + +#ifndef CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT +CK_PR_N_S(neg, int, int, -) +#endif /* CK_F_PR_NEG_INT */ + +#ifndef CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_INT_ZERO +CK_PR_N_Z_S(int, int) +#endif /* CK_F_PR_NEG_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_NEG_DOUBLE +#define CK_F_PR_NEG_DOUBLE +CK_PR_N_S(neg, double, double, -) +#endif /* CK_F_PR_NEG_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_NOT_UINT +#define CK_F_PR_NOT_UINT +CK_PR_N_S(not, uint, unsigned int, ~) +#endif /* CK_F_PR_NOT_UINT */ + +#ifndef CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT +CK_PR_N_S(neg, uint, unsigned int, -) +#endif /* CK_F_PR_NEG_UINT */ + +#ifndef CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NEG_UINT_ZERO +CK_PR_N_Z_S(uint, unsigned int) +#endif /* CK_F_PR_NEG_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_PTR +CK_PR_N(not, ptr, void, uintptr_t, ~, void *) +#endif /* CK_F_PR_NOT_PTR */ + +#ifndef CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR +CK_PR_N(neg, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_NEG_PTR */ + +#ifndef CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_PTR_ZERO +CK_PR_N_Z(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_NEG_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_NOT_64 +#define CK_F_PR_NOT_64 +CK_PR_N_S(not, 64, uint64_t, ~) +#endif /* CK_F_PR_NOT_64 */ + +#ifndef CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64 +CK_PR_N_S(neg, 64, uint64_t, -) +#endif /* CK_F_PR_NEG_64 */ + +#ifndef CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_64_ZERO +CK_PR_N_Z_S(64, uint64_t) +#endif /* CK_F_PR_NEG_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_NOT_32 +#define CK_F_PR_NOT_32 +CK_PR_N_S(not, 32, uint32_t, ~) +#endif /* CK_F_PR_NOT_32 */ + +#ifndef CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32 +CK_PR_N_S(neg, 32, uint32_t, -) +#endif /* CK_F_PR_NEG_32 */ + +#ifndef CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_32_ZERO +CK_PR_N_Z_S(32, uint32_t) +#endif /* CK_F_PR_NEG_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_NOT_16 +#define CK_F_PR_NOT_16 +CK_PR_N_S(not, 16, uint16_t, ~) +#endif /* CK_F_PR_NOT_16 */ + +#ifndef CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16 +CK_PR_N_S(neg, 16, uint16_t, -) +#endif /* CK_F_PR_NEG_16 */ + +#ifndef CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_16_ZERO +CK_PR_N_Z_S(16, uint16_t) +#endif /* CK_F_PR_NEG_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_NOT_8 +#define CK_F_PR_NOT_8 +CK_PR_N_S(not, 8, uint8_t, ~) +#endif /* CK_F_PR_NOT_8 */ + +#ifndef CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8 +CK_PR_N_S(neg, 8, uint8_t, -) +#endif /* CK_F_PR_NEG_8 */ + +#ifndef CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_8_ZERO +CK_PR_N_Z_S(8, uint8_t) +#endif /* CK_F_PR_NEG_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_N_Z_S +#undef CK_PR_N_S +#undef CK_PR_N_Z +#undef CK_PR_N + +#define CK_PR_FAA(S, M, T, C) \ + CK_CC_INLINE static C \ + ck_pr_faa_##S(M *target, T delta) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous + delta), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return ((C)previous); \ + } + +#define CK_PR_FAS(S, M, C) \ + CK_CC_INLINE static C \ + ck_pr_fas_##S(M *target, C update) \ + { \ + C previous; \ + previous = ck_pr_md_load_##S(target); \ + while (ck_pr_cas_##S##_value(target, \ + previous, \ + update, \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return (previous); \ + } + +#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) +#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_CHAR +CK_PR_FAA_S(char, char) +#endif /* CK_F_PR_FAA_CHAR */ + +#ifndef CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_CHAR +CK_PR_FAS_S(char, char) +#endif /* CK_F_PR_FAS_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_FAA_INT +#define CK_F_PR_FAA_INT +CK_PR_FAA_S(int, int) +#endif /* CK_F_PR_FAA_INT */ + +#ifndef CK_F_PR_FAS_INT +#define CK_F_PR_FAS_INT +CK_PR_FAS_S(int, int) +#endif /* CK_F_PR_FAS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_FAA_DOUBLE +#define CK_F_PR_FAA_DOUBLE +CK_PR_FAA_S(double, double) +#endif /* CK_F_PR_FAA_DOUBLE */ + +#ifndef CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FAS_DOUBLE +CK_PR_FAS_S(double, double) +#endif /* CK_F_PR_FAS_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_FAA_UINT +#define CK_F_PR_FAA_UINT +CK_PR_FAA_S(uint, unsigned int) +#endif /* CK_F_PR_FAA_UINT */ + +#ifndef CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_UINT +CK_PR_FAS_S(uint, unsigned int) +#endif /* CK_F_PR_FAS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_PTR +CK_PR_FAA(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_FAA_PTR */ + +#ifndef CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_PTR +CK_PR_FAS(ptr, void, void *) +#endif /* CK_F_PR_FAS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_FAA_64 +#define CK_F_PR_FAA_64 +CK_PR_FAA_S(64, uint64_t) +#endif /* CK_F_PR_FAA_64 */ + +#ifndef CK_F_PR_FAS_64 +#define CK_F_PR_FAS_64 +CK_PR_FAS_S(64, uint64_t) +#endif /* CK_F_PR_FAS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_FAA_32 +#define CK_F_PR_FAA_32 +CK_PR_FAA_S(32, uint32_t) +#endif /* CK_F_PR_FAA_32 */ + +#ifndef CK_F_PR_FAS_32 +#define CK_F_PR_FAS_32 +CK_PR_FAS_S(32, uint32_t) +#endif /* CK_F_PR_FAS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_FAA_16 +#define CK_F_PR_FAA_16 +CK_PR_FAA_S(16, uint16_t) +#endif /* CK_F_PR_FAA_16 */ + +#ifndef CK_F_PR_FAS_16 +#define CK_F_PR_FAS_16 +CK_PR_FAS_S(16, uint16_t) +#endif /* CK_F_PR_FAS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_FAA_8 +#define CK_F_PR_FAA_8 +CK_PR_FAA_S(8, uint8_t) +#endif /* CK_F_PR_FAA_8 */ + +#ifndef CK_F_PR_FAS_8 +#define CK_F_PR_FAS_8 +CK_PR_FAS_S(8, uint8_t) +#endif /* CK_F_PR_FAS_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_FAA_S +#undef CK_PR_FAS_S +#undef CK_PR_FAA +#undef CK_PR_FAS + +#endif /* CK_PR_H */ diff --git a/include/ck_queue.h b/include/ck_queue.h new file mode 100644 index 0000000..c1e9872 --- /dev/null +++ b/include/ck_queue.h @@ -0,0 +1,428 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/*- + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + * $FreeBSD: release/9.0.0/sys/sys/queue.h 221843 2011-05-13 15:49:23Z mdf $ + */ + +#ifndef CK_QUEUE_H +#define CK_QUEUE_H + +#include <ck_pr.h> + +/* + * This file defines three types of data structures: singly-linked lists, + * singly-linked tail queues and lists. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * It is safe to use _FOREACH/_FOREACH_SAFE in the presence of concurrent + * modifications to the list. Writers to these lists must, on the other hand, + * implement writer-side synchronization. The _SWAP operations are not atomic. + * This facility is currently unsupported on architectures such as the Alpha + * which require load-depend memory fences. + * + * CK_SLIST CK_LIST CK_STAILQ + * _HEAD + + + + * _HEAD_INITIALIZER + + + + * _ENTRY + + + + * _INIT + + + + * _EMPTY + + + + * _FIRST + + + + * _NEXT + + + + * _FOREACH + + + + * _FOREACH_SAFE + + + + * _INSERT_HEAD + + + + * _INSERT_BEFORE - + - + * _INSERT_AFTER + + + + * _INSERT_TAIL - - + + * _REMOVE_AFTER + - + + * _REMOVE_HEAD + - + + * _REMOVE + + + + * _SWAP + + + + * _MOVE + + + + */ + +/* + * Singly-linked List declarations. + */ +#define CK_SLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define CK_SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +/* + * Singly-linked List functions. + */ +#define CK_SLIST_EMPTY(head) \ + (ck_pr_load_ptr(&(head)->slh_first) == NULL) + +#define CK_SLIST_FIRST(head) \ + (ck_pr_load_ptr(&(head)->slh_first)) + +#define CK_SLIST_NEXT(elm, field) \ + ck_pr_load_ptr(&((elm)->field.sle_next)) + +#define CK_SLIST_FOREACH(var, head, field) \ + for ((var) = CK_SLIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_SLIST_NEXT((var), field)) + +#define CK_SLIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_SLIST_FIRST(head); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_SLIST_NEXT(var, field), 1);\ + (var) = (tvar)) + +#define CK_SLIST_FOREACH_PREVPTR(var, varp, head, field) \ + for ((varp) = &(head)->slh_first; \ + ((var) = ck_pr_load_ptr(varp)) != NULL && (ck_pr_fence_load(), 1); \ + (varp) = &(var)->field.sle_next) + +#define CK_SLIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->slh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_SLIST_INSERT_AFTER(a, b, field) do { \ + (b)->field.sle_next = (a)->field.sle_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(a)->field.sle_next, b); \ +} while (0) + +#define CK_SLIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.sle_next = (head)->slh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->slh_first, elm); \ +} while (0) + +#define CK_SLIST_REMOVE_AFTER(elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.sle_next, \ + (elm)->field.sle_next->field.sle_next); \ +} while (0) + +#define CK_SLIST_REMOVE(head, elm, type, field) do { \ + if ((head)->slh_first == (elm)) { \ + CK_SLIST_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->slh_first; \ + while (curelm->field.sle_next != (elm)) \ + curelm = curelm->field.sle_next; \ + CK_SLIST_REMOVE_AFTER(curelm, field); \ + } \ +} while (0) + +#define CK_SLIST_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->slh_first, \ + (head)->slh_first->field.sle_next); \ +} while (0) + +#define CK_SLIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->slh_first, (head2)->slh_first); \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_SLIST_SWAP(a, b, type) do { \ + struct type *swap_first = (a)->slh_first; \ + (a)->slh_first = (b)->slh_first; \ + (b)->slh_first = swap_first; \ +} while (0) + +/* + * Singly-linked Tail queue declarations. + */ +#define CK_STAILQ_HEAD(name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ + struct type **stqh_last;/* addr of last next element */ \ +} + +#define CK_STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +#define CK_STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +/* + * Singly-linked Tail queue functions. + */ +#define CK_STAILQ_CONCAT(head1, head2) do { \ + if ((head2)->stqh_first == NULL) { \ + ck_pr_store_ptr((head1)->stqh_last, (head2)->stqh_first); \ + ck_pr_fence_store(); \ + (head1)->stqh_last = (head2)->stqh_last; \ + CK_STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define CK_STAILQ_EMPTY(head) (ck_pr_load_ptr(&(head)->stqh_first) == NULL) + +#define CK_STAILQ_FIRST(head) (ck_pr_load_ptr(&(head)->stqh_first)) + +#define CK_STAILQ_FOREACH(var, head, field) \ + for((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_STAILQ_NEXT((var), field)) + +#define CK_STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = \ + CK_STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define CK_STAILQ_INIT(head) do { \ + ck_pr_store_ptr(&(head)->stqh_first, NULL); \ + ck_pr_fence_store(); \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (0) + +#define CK_STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ + (elm)->field.stqe_next = (tqelm)->field.stqe_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(tqelm)->field.stqe_next, elm); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.stqe_next = (head)->stqh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->stqh_first, elm); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.stqe_next = NULL; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((head)->stqh_last, (elm)); \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_NEXT(elm, field) \ + (ck_pr_load_ptr(&(elm)->field.stqe_next)) + +#define CK_STAILQ_REMOVE(head, elm, type, field) do { \ + if ((head)->stqh_first == (elm)) { \ + CK_STAILQ_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->stqh_first; \ + while (curelm->field.stqe_next != (elm)) \ + curelm = curelm->field.stqe_next; \ + CK_STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ +} while (0) + +#define CK_STAILQ_REMOVE_AFTER(head, elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.stqe_next, \ + (elm)->field.stqe_next->field.stqe_next); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->stqh_first, \ + (head)->stqh_first->field.stqe_next); \ + if ((head)->stqh_first == NULL) \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (0) + +#define CK_STAILQ_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->stqh_first, (head2)->stqh_first); \ + (head1)->stqh_last = (head2)->stqh_last; \ + if ((head2)->stqh_last == &(head2)->stqh_first) \ + (head1)->stqh_last = &(head1)->stqh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_STAILQ_SWAP(head1, head2, type) do { \ + struct type *swap_first = CK_STAILQ_FIRST(head1); \ + struct type **swap_last = (head1)->stqh_last; \ + CK_STAILQ_FIRST(head1) = CK_STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + CK_STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (CK_STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &(head1)->stqh_first; \ + if (CK_STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &(head2)->stqh_first; \ +} while (0) + +/* + * List declarations. + */ +#define CK_LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define CK_LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +#define CK_LIST_FIRST(head) ck_pr_load_ptr(&(head)->lh_first) +#define CK_LIST_EMPTY(head) (CK_LIST_FIRST(head) == NULL) +#define CK_LIST_NEXT(elm, field) ck_pr_load_ptr(&(elm)->field.le_next) + +#define CK_LIST_FOREACH(var, head, field) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_LIST_NEXT((var), field)) + +#define CK_LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_LIST_NEXT((var), field), 1);\ + (var) = (tvar)) + +#define CK_LIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->lh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_LIST_INSERT_AFTER(listelm, elm, field) do { \ + (elm)->field.le_next = (listelm)->field.le_next; \ + (elm)->field.le_prev = &(listelm)->field.le_next; \ + ck_pr_fence_store(); \ + if ((listelm)->field.le_next != NULL) \ + (listelm)->field.le_next->field.le_prev = &(elm)->field.le_next;\ + ck_pr_store_ptr(&(listelm)->field.le_next, elm); \ +} while (0) + +#define CK_LIST_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + (elm)->field.le_next = (listelm); \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((listelm)->field.le_prev, (elm)); \ + (listelm)->field.le_prev = &(elm)->field.le_next; \ +} while (0) + +#define CK_LIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.le_next = (head)->lh_first; \ + ck_pr_fence_store(); \ + if ((elm)->field.le_next != NULL) \ + (head)->lh_first->field.le_prev = &(elm)->field.le_next; \ + ck_pr_store_ptr(&(head)->lh_first, elm); \ + (elm)->field.le_prev = &(head)->lh_first; \ +} while (0) + +#define CK_LIST_REMOVE(elm, field) do { \ + ck_pr_store_ptr((elm)->field.le_prev, (elm)->field.le_next); \ + if ((elm)->field.le_next != NULL) \ + (elm)->field.le_next->field.le_prev = (elm)->field.le_prev; \ +} while (0) + +#define CK_LIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->lh_first, (head2)->lh_first); \ + if ((head1)->lh_first != NULL) \ + (head1)->lh_first->field.le_prev = &(head1)->lh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_LIST_SWAP(head1, head2, type, field) do { \ + struct type *swap_tmp = (head1)->lh_first; \ + (head1)->lh_first = (head2)->lh_first; \ + (head2)->lh_first = swap_tmp; \ + if ((swap_tmp = (head1)->lh_first) != NULL) \ + swap_tmp->field.le_prev = &(head1)->lh_first; \ + if ((swap_tmp = (head2)->lh_first) != NULL) \ + swap_tmp->field.le_prev = &(head2)->lh_first; \ +} while (0) + +#endif /* CK_QUEUE_H */ diff --git a/include/ck_rhs.h b/include/ck_rhs.h new file mode 100644 index 0000000..2a21a73 --- /dev/null +++ b/include/ck_rhs.h @@ -0,0 +1,134 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RHS_H +#define CK_RHS_H + +#include <ck_cc.h> +#include <ck_malloc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdint.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +/* + * Indicates a single-writer many-reader workload. Mutually + * exclusive with CK_RHS_MODE_MPMC + */ +#define CK_RHS_MODE_SPMC 1 + +/* + * Indicates that values to be stored are not pointers but + * values. Allows for full precision. Mutually exclusive + * with CK_RHS_MODE_OBJECT. + */ +#define CK_RHS_MODE_DIRECT 2 + +/* + * Indicates that the values to be stored are pointers. + * Allows for space optimizations in the presence of pointer + * packing. Mutually exclusive with CK_RHS_MODE_DIRECT. + */ +#define CK_RHS_MODE_OBJECT 8 + +/* + * Indicated that the load is read-mostly, so get should be optimized + * over put and delete + */ +#define CK_RHS_MODE_READ_MOSTLY 16 + +/* Currently unsupported. */ +#define CK_RHS_MODE_MPMC (void) + +/* + * Hash callback function. + */ +typedef unsigned long ck_rhs_hash_cb_t(const void *, unsigned long); + +/* + * Returns pointer to object if objects are equivalent. + */ +typedef bool ck_rhs_compare_cb_t(const void *, const void *); + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_RHS_PP +#define CK_RHS_KEY_MASK ((1U << ((sizeof(void *) * 8) - CK_MD_VMA_BITS)) - 1) +#endif + +struct ck_rhs_map; +struct ck_rhs { + struct ck_malloc *m; + struct ck_rhs_map *map; + unsigned int mode; + unsigned int load_factor; + unsigned long seed; + ck_rhs_hash_cb_t *hf; + ck_rhs_compare_cb_t *compare; +}; +typedef struct ck_rhs ck_rhs_t; + +struct ck_rhs_stat { + unsigned long n_entries; + unsigned int probe_maximum; +}; + +struct ck_rhs_iterator { + void **cursor; + unsigned long offset; +}; +typedef struct ck_rhs_iterator ck_rhs_iterator_t; + +#define CK_RHS_ITERATOR_INITIALIZER { NULL, 0 } + +/* Convenience wrapper to table hash function. */ +#define CK_RHS_HASH(T, F, K) F((K), (T)->seed) + +typedef void *ck_rhs_apply_fn_t(void *, void *); +bool ck_rhs_apply(ck_rhs_t *, unsigned long, const void *, ck_rhs_apply_fn_t *, void *); +void ck_rhs_iterator_init(ck_rhs_iterator_t *); +bool ck_rhs_next(ck_rhs_t *, ck_rhs_iterator_t *, void **); +bool ck_rhs_move(ck_rhs_t *, ck_rhs_t *, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *); +bool ck_rhs_init(ck_rhs_t *, unsigned int, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *, unsigned long, unsigned long); +void ck_rhs_destroy(ck_rhs_t *); +void *ck_rhs_get(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put_unique(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_set(ck_rhs_t *, unsigned long, const void *, void **); +bool ck_rhs_fas(ck_rhs_t *, unsigned long, const void *, void **); +void *ck_rhs_remove(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_grow(ck_rhs_t *, unsigned long); +bool ck_rhs_rebuild(ck_rhs_t *); +bool ck_rhs_gc(ck_rhs_t *); +unsigned long ck_rhs_count(ck_rhs_t *); +bool ck_rhs_reset(ck_rhs_t *); +bool ck_rhs_reset_size(ck_rhs_t *, unsigned long); +void ck_rhs_stat(ck_rhs_t *, struct ck_rhs_stat *); +bool ck_rhs_set_load_factor(ck_rhs_t *, unsigned int); + +#endif /* CK_RHS_H */ diff --git a/include/ck_ring.h b/include/ck_ring.h new file mode 100644 index 0000000..8a2a791 --- /dev/null +++ b/include/ck_ring.h @@ -0,0 +1,656 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RING_H +#define CK_RING_H + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_string.h> + +/* + * Concurrent ring buffer. + */ + +struct ck_ring { + unsigned int c_head; + char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; + unsigned int p_tail; + unsigned int p_head; + char _pad[CK_MD_CACHELINE - sizeof(unsigned int) * 2]; + unsigned int size; + unsigned int mask; +}; +typedef struct ck_ring ck_ring_t; + +struct ck_ring_buffer { + void *value; +}; +typedef struct ck_ring_buffer ck_ring_buffer_t; + +CK_CC_INLINE static unsigned int +ck_ring_size(const struct ck_ring *ring) +{ + unsigned int c, p; + + c = ck_pr_load_uint(&ring->c_head); + p = ck_pr_load_uint(&ring->p_tail); + return (p - c) & ring->mask; +} + +CK_CC_INLINE static unsigned int +ck_ring_capacity(const struct ck_ring *ring) +{ + return ring->size; +} + +CK_CC_INLINE static void +ck_ring_init(struct ck_ring *ring, unsigned int size) +{ + + ring->size = size; + ring->mask = size - 1; + ring->p_tail = 0; + ring->p_head = 0; + ring->c_head = 0; + return; +} + +/* + * The _ck_ring_* namespace is internal only and must not used externally. + */ +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_sp(struct ck_ring *ring, + void *CK_CC_RESTRICT buffer, + const void *CK_CC_RESTRICT entry, + unsigned int ts, + unsigned int *size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer, delta; + + consumer = ck_pr_load_uint(&ring->c_head); + producer = ring->p_tail; + delta = producer + 1; + if (size != NULL) + *size = (producer - consumer) & mask; + + if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask))) + return false; + + buffer = (char *)buffer + ts * (producer & mask); + memcpy(buffer, entry, ts); + + /* + * Make sure to update slot value before indicating + * that the slot is available for consumption. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->p_tail, delta); + return true; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_sp_size(struct ck_ring *ring, + void *CK_CC_RESTRICT buffer, + const void *CK_CC_RESTRICT entry, + unsigned int ts, + unsigned int *size) +{ + unsigned int sz; + bool r; + + r = _ck_ring_enqueue_sp(ring, buffer, entry, ts, &sz); + *size = sz; + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_dequeue_sc(struct ck_ring *ring, + const void *CK_CC_RESTRICT buffer, + void *CK_CC_RESTRICT target, + unsigned int size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ring->c_head; + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + /* + * Make sure to serialize with respect to our snapshot + * of the producer counter. + */ + ck_pr_fence_load(); + + buffer = (const char *)buffer + size * (consumer & mask); + memcpy(target, buffer, size); + + /* + * Make sure copy is completed with respect to consumer + * update. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->c_head, consumer + 1); + return true; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_mp(struct ck_ring *ring, + void *buffer, + const void *entry, + unsigned int ts, + unsigned int *size) +{ + const unsigned int mask = ring->mask; + unsigned int producer, consumer, delta; + bool r = true; + + producer = ck_pr_load_uint(&ring->p_head); + + do { + /* + * The snapshot of producer must be up to date with + * respect to consumer. + */ + ck_pr_fence_load(); + consumer = ck_pr_load_uint(&ring->c_head); + + delta = producer + 1; + if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask))) { + r = false; + goto leave; + } + } while (ck_pr_cas_uint_value(&ring->p_head, + producer, + delta, + &producer) == false); + + buffer = (char *)buffer + ts * (producer & mask); + memcpy(buffer, entry, ts); + + /* + * Wait until all concurrent producers have completed writing + * their data into the ring buffer. + */ + while (ck_pr_load_uint(&ring->p_tail) != producer) + ck_pr_stall(); + + /* + * Ensure that copy is completed before updating shared producer + * counter. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->p_tail, delta); + +leave: + if (size != NULL) + *size = (producer - consumer) & mask; + + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_mp_size(struct ck_ring *ring, + void *buffer, + const void *entry, + unsigned int ts, + unsigned int *size) +{ + unsigned int sz; + bool r; + + r = _ck_ring_enqueue_mp(ring, buffer, entry, ts, &sz); + *size = sz; + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_trydequeue_mc(struct ck_ring *ring, + const void *buffer, + void *data, + unsigned int size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ck_pr_load_uint(&ring->c_head); + ck_pr_fence_load(); + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + ck_pr_fence_load(); + + buffer = (const char *)buffer + size * (consumer & mask); + memcpy(data, buffer, size); + + ck_pr_fence_store_atomic(); + return ck_pr_cas_uint(&ring->c_head, consumer, consumer + 1); +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_dequeue_mc(struct ck_ring *ring, + const void *buffer, + void *data, + unsigned int ts) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ck_pr_load_uint(&ring->c_head); + + do { + const char *target; + + /* + * Producer counter must represent state relative to + * our latest consumer snapshot. + */ + ck_pr_fence_load(); + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + ck_pr_fence_load(); + + target = (const char *)buffer + ts * (consumer & mask); + memcpy(data, target, ts); + + /* Serialize load with respect to head update. */ + ck_pr_fence_store_atomic(); + } while (ck_pr_cas_uint_value(&ring->c_head, + consumer, + consumer + 1, + &consumer) == false); + + return true; +} + +/* + * The ck_ring_*_spsc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is only provided if there is up + * to one concurrent consumer and up to one concurrent producer. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_spsc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_sp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_spsc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_sp(ring, buffer, + &entry, sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_spsc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_sc(ring, buffer, + (void **)data, sizeof(void *)); +} + +/* + * The ck_ring_*_mpmc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * producers and consumers. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_mpmc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_mp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_mpmc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_mp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_trydequeue_mpmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_trydequeue_mc(ring, + buffer, (void **)data, sizeof(void *)); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_mpmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_mc(ring, buffer, (void **)data, + sizeof(void *)); +} + +/* + * The ck_ring_*_spmc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * consumers with up to one concurrent producer. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_spmc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_sp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_spmc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_sp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_trydequeue_spmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_trydequeue_mc(ring, buffer, (void **)data, sizeof(void *)); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_spmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_mc(ring, buffer, (void **)data, sizeof(void *)); +} + +/* + * The ck_ring_*_mpsc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * producers with up to one concurrent consumers. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_mpsc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_mp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_mpsc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_mp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_mpsc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_sc(ring, buffer, (void **)data, + sizeof(void *)); +} + +/* + * CK_RING_PROTOTYPE is used to define a type-safe interface for inlining + * values of a particular type in the ring the buffer. + */ +#define CK_RING_PROTOTYPE(name, type) \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spsc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_sp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_sp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_spsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_sc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spmc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_sp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_sp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_trydequeue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_trydequeue_mc(a, \ + b, c, sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_mc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_mp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpsc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_mp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_mpsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_sc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpmc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_mp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_mp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_trydequeue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_trydequeue_mc(a, \ + b, c, sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_mc(a, b, c, \ + sizeof(struct type)); \ +} + +/* + * A single producer with one concurrent consumer. + */ +#define CK_RING_ENQUEUE_SPSC(name, a, b, c) \ + ck_ring_enqueue_spsc_##name(a, b, c) +#define CK_RING_ENQUEUE_SPSC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_spsc_size_##name(a, b, c, d) +#define CK_RING_DEQUEUE_SPSC(name, a, b, c) \ + ck_ring_dequeue_spsc_##name(a, b, c) + +/* + * A single producer with any number of concurrent consumers. + */ +#define CK_RING_ENQUEUE_SPMC(name, a, b, c) \ + ck_ring_enqueue_spmc_##name(a, b, c) +#define CK_RING_ENQUEUE_SPMC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_spmc_size_##name(a, b, c, d) +#define CK_RING_TRYDEQUEUE_SPMC(name, a, b, c) \ + ck_ring_trydequeue_spmc_##name(a, b, c) +#define CK_RING_DEQUEUE_SPMC(name, a, b, c) \ + ck_ring_dequeue_spmc_##name(a, b, c) + +/* + * Any number of concurrent producers with up to one + * concurrent consumer. + */ +#define CK_RING_ENQUEUE_MPSC(name, a, b, c) \ + ck_ring_enqueue_mpsc_##name(a, b, c) +#define CK_RING_ENQUEUE_MPSC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_mpsc_size_##name(a, b, c, d) +#define CK_RING_DEQUEUE_MPSC(name, a, b, c) \ + ck_ring_dequeue_mpsc_##name(a, b, c) + +/* + * Any number of concurrent producers and consumers. + */ +#define CK_RING_ENQUEUE_MPMC(name, a, b, c) \ + ck_ring_enqueue_mpmc_##name(a, b, c) +#define CK_RING_ENQUEUE_MPMC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_mpmc_size_##name(a, b, c, d) +#define CK_RING_TRYDEQUEUE_MPMC(name, a, b, c) \ + ck_ring_trydequeue_mpmc_##name(a, b, c) +#define CK_RING_DEQUEUE_MPMC(name, a, b, c) \ + ck_ring_dequeue_mpmc_##name(a, b, c) + +#endif /* CK_RING_H */ diff --git a/include/ck_rwcohort.h b/include/ck_rwcohort.h new file mode 100644 index 0000000..7519b5d --- /dev/null +++ b/include/ck_rwcohort.h @@ -0,0 +1,317 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * Copyright 2013 Brendon Scheinman. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RWCOHORT_H +#define CK_RWCOHORT_H + +/* + * This is an implementation of NUMA-aware reader-writer locks as described in: + * Calciu, I.; Dice, D.; Lev, Y.; Luchangco, V.; Marathe, V.; and Shavit, N. 2014. + * NUMA-Aware Reader-Writer Locks + */ + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stddef.h> +#include <ck_cohort.h> + +#define CK_RWCOHORT_WP_NAME(N) ck_rwcohort_wp_##N +#define CK_RWCOHORT_WP_INSTANCE(N) struct CK_RWCOHORT_WP_NAME(N) +#define CK_RWCOHORT_WP_INIT(N, RW, WL) ck_rwcohort_wp_##N##_init(RW, WL) +#define CK_RWCOHORT_WP_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_read_unlock(RW) +#define CK_RWCOHORT_WP_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_WP_PROTOTYPE(N) \ + CK_RWCOHORT_WP_INSTANCE(N) { \ + unsigned int read_counter; \ + unsigned int write_barrier; \ + unsigned int wait_limit; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_init(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + unsigned int wait_limit) \ + { \ + \ + rw_cohort->read_counter = 0; \ + rw_cohort->write_barrier = 0; \ + rw_cohort->wait_limit = wait_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_write_lock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + while (ck_pr_load_uint(&rw_cohort->write_barrier) > 0) \ + ck_pr_stall(); \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) \ + ck_pr_stall(); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_write_unlock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_read_lock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + unsigned int wait_count = 0; \ + bool raised = false; \ + \ + for (;;) { \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + ck_pr_fence_atomic_load(); \ + if (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == false) \ + break; \ + \ + ck_pr_dec_uint(&rw_cohort->read_counter); \ + while (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == true) { \ + ck_pr_stall(); \ + if (++wait_count > rw_cohort->wait_limit && \ + raised == false) { \ + ck_pr_inc_uint(&rw_cohort->write_barrier); \ + raised = true; \ + } \ + } \ + } \ + \ + if (raised == true) \ + ck_pr_dec_uint(&rw_cohort->write_barrier); \ + \ + ck_pr_fence_load(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_read_unlock(CK_RWCOHORT_WP_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_WP_INITIALIZER { \ + .read_counter = 0, \ + .write_barrier = 0, \ + .wait_limit = 0 \ +} + +#define CK_RWCOHORT_RP_NAME(N) ck_rwcohort_rp_##N +#define CK_RWCOHORT_RP_INSTANCE(N) struct CK_RWCOHORT_RP_NAME(N) +#define CK_RWCOHORT_RP_INIT(N, RW, WL) ck_rwcohort_rp_##N##_init(RW, WL) +#define CK_RWCOHORT_RP_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_read_unlock(RW) +#define CK_RWCOHORT_RP_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_RP_PROTOTYPE(N) \ + CK_RWCOHORT_RP_INSTANCE(N) { \ + unsigned int read_counter; \ + unsigned int read_barrier; \ + unsigned int wait_limit; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_init(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + unsigned int wait_limit) \ + { \ + \ + rw_cohort->read_counter = 0; \ + rw_cohort->read_barrier = 0; \ + rw_cohort->wait_limit = wait_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_write_lock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + unsigned int wait_count = 0; \ + bool raised = false; \ + \ + for (;;) { \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + if (ck_pr_load_uint(&rw_cohort->read_counter) == 0) \ + break; \ + \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) { \ + ck_pr_stall(); \ + if (++wait_count > rw_cohort->wait_limit && \ + raised == false) { \ + ck_pr_inc_uint(&rw_cohort->read_barrier); \ + raised = true; \ + } \ + } \ + } \ + \ + if (raised == true) \ + ck_pr_dec_uint(&rw_cohort->read_barrier); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_write_unlock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_read_lock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + while (ck_pr_load_uint(&rw_cohort->read_barrier) > 0) \ + ck_pr_stall(); \ + \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + ck_pr_fence_atomic_load(); \ + \ + while (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == true) \ + ck_pr_stall(); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_read_unlock(CK_RWCOHORT_RP_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_RP_INITIALIZER { \ + .read_counter = 0, \ + .read_barrier = 0, \ + .wait_limit = 0 \ +} + +#define CK_RWCOHORT_NEUTRAL_NAME(N) ck_rwcohort_neutral_##N +#define CK_RWCOHORT_NEUTRAL_INSTANCE(N) struct CK_RWCOHORT_NEUTRAL_NAME(N) +#define CK_RWCOHORT_NEUTRAL_INIT(N, RW) ck_rwcohort_neutral_##N##_init(RW) +#define CK_RWCOHORT_NEUTRAL_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_read_unlock(RW) +#define CK_RWCOHORT_NEUTRAL_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_NEUTRAL_PROTOTYPE(N) \ + CK_RWCOHORT_NEUTRAL_INSTANCE(N) { \ + unsigned int read_counter; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_init(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort) \ + { \ + \ + rw_cohort->read_counter = 0; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_write_lock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort,\ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) { \ + ck_pr_stall(); \ + } \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_write_unlock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort,\ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_read_lock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_read_unlock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_NEUTRAL_INITIALIZER { \ + .read_counter = 0, \ +} + +#endif /* CK_RWCOHORT_H */ diff --git a/include/ck_rwlock.h b/include/ck_rwlock.h new file mode 100644 index 0000000..b82b4b5 --- /dev/null +++ b/include/ck_rwlock.h @@ -0,0 +1,302 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RWLOCK_H +#define CK_RWLOCK_H + +#include <ck_elide.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_rwlock { + unsigned int writer; + unsigned int n_readers; +}; +typedef struct ck_rwlock ck_rwlock_t; + +#define CK_RWLOCK_INITIALIZER {0, 0} + +CK_CC_INLINE static void +ck_rwlock_init(struct ck_rwlock *rw) +{ + + rw->writer = 0; + rw->n_readers = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_rwlock_write_unlock(ck_rwlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&rw->writer, 0); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked_writer(ck_rwlock_t *rw) +{ + bool r; + + r = ck_pr_load_uint(&rw->writer); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_rwlock_write_downgrade(ck_rwlock_t *rw) +{ + + ck_pr_inc_uint(&rw->n_readers); + ck_rwlock_write_unlock(rw); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked(ck_rwlock_t *rw) +{ + bool l; + + l = ck_pr_load_uint(&rw->n_readers) | + ck_pr_load_uint(&rw->writer); + ck_pr_fence_acquire(); + return l; +} + +CK_CC_INLINE static bool +ck_rwlock_write_trylock(ck_rwlock_t *rw) +{ + + if (ck_pr_fas_uint(&rw->writer, 1) != 0) + return false; + + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->n_readers) != 0) { + ck_rwlock_write_unlock(rw); + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_rwlock_write, ck_rwlock_t, + ck_rwlock_locked, ck_rwlock_write_trylock) + +CK_CC_INLINE static void +ck_rwlock_write_lock(ck_rwlock_t *rw) +{ + + while (ck_pr_fas_uint(&rw->writer, 1) != 0) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + while (ck_pr_load_uint(&rw->n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_ELIDE_PROTOTYPE(ck_rwlock_write, ck_rwlock_t, + ck_rwlock_locked, ck_rwlock_write_lock, + ck_rwlock_locked_writer, ck_rwlock_write_unlock) + +CK_CC_INLINE static bool +ck_rwlock_read_trylock(ck_rwlock_t *rw) +{ + + if (ck_pr_load_uint(&rw->writer) != 0) + return false; + + ck_pr_inc_uint(&rw->n_readers); + + /* + * Serialize with respect to concurrent write + * lock operation. + */ + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->writer) == 0) { + ck_pr_fence_lock(); + return true; + } + + ck_pr_dec_uint(&rw->n_readers); + return false; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_rwlock_read, ck_rwlock_t, + ck_rwlock_locked_writer, ck_rwlock_read_trylock) + +CK_CC_INLINE static void +ck_rwlock_read_lock(ck_rwlock_t *rw) +{ + + for (;;) { + while (ck_pr_load_uint(&rw->writer) != 0) + ck_pr_stall(); + + ck_pr_inc_uint(&rw->n_readers); + + /* + * Serialize with respect to concurrent write + * lock operation. + */ + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->writer) == 0) + break; + + ck_pr_dec_uint(&rw->n_readers); + } + + /* Acquire semantics are necessary. */ + ck_pr_fence_load(); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked_reader(ck_rwlock_t *rw) +{ + + ck_pr_fence_load(); + return ck_pr_load_uint(&rw->n_readers); +} + +CK_CC_INLINE static void +ck_rwlock_read_unlock(ck_rwlock_t *rw) +{ + + ck_pr_fence_load_atomic(); + ck_pr_dec_uint(&rw->n_readers); + return; +} + +CK_ELIDE_PROTOTYPE(ck_rwlock_read, ck_rwlock_t, + ck_rwlock_locked_writer, ck_rwlock_read_lock, + ck_rwlock_locked_reader, ck_rwlock_read_unlock) + +/* + * Recursive writer reader-writer lock implementation. + */ +struct ck_rwlock_recursive { + struct ck_rwlock rw; + unsigned int wc; +}; +typedef struct ck_rwlock_recursive ck_rwlock_recursive_t; + +#define CK_RWLOCK_RECURSIVE_INITIALIZER {CK_RWLOCK_INITIALIZER, 0} + +CK_CC_INLINE static void +ck_rwlock_recursive_write_lock(ck_rwlock_recursive_t *rw, unsigned int tid) +{ + unsigned int o; + + o = ck_pr_load_uint(&rw->rw.writer); + if (o == tid) + goto leave; + + while (ck_pr_cas_uint(&rw->rw.writer, 0, tid) == false) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + while (ck_pr_load_uint(&rw->rw.n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); +leave: + rw->wc++; + return; +} + +CK_CC_INLINE static bool +ck_rwlock_recursive_write_trylock(ck_rwlock_recursive_t *rw, unsigned int tid) +{ + unsigned int o; + + o = ck_pr_load_uint(&rw->rw.writer); + if (o == tid) + goto leave; + + if (ck_pr_cas_uint(&rw->rw.writer, 0, tid) == false) + return false; + + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->rw.n_readers) != 0) { + ck_pr_store_uint(&rw->rw.writer, 0); + return false; + } + + ck_pr_fence_lock(); +leave: + rw->wc++; + return true; +} + +CK_CC_INLINE static void +ck_rwlock_recursive_write_unlock(ck_rwlock_recursive_t *rw) +{ + + if (--rw->wc == 0) { + ck_pr_fence_unlock(); + ck_pr_store_uint(&rw->rw.writer, 0); + } + + return; +} + +CK_CC_INLINE static void +ck_rwlock_recursive_read_lock(ck_rwlock_recursive_t *rw) +{ + + ck_rwlock_read_lock(&rw->rw); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_recursive_read_trylock(ck_rwlock_recursive_t *rw) +{ + + return ck_rwlock_read_trylock(&rw->rw); +} + +CK_CC_INLINE static void +ck_rwlock_recursive_read_unlock(ck_rwlock_recursive_t *rw) +{ + + ck_rwlock_read_unlock(&rw->rw); + return; +} + +#endif /* CK_RWLOCK_H */ diff --git a/include/ck_sequence.h b/include/ck_sequence.h new file mode 100644 index 0000000..6a482b9 --- /dev/null +++ b/include/ck_sequence.h @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SEQUENCE_H +#define CK_SEQUENCE_H + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +struct ck_sequence { + unsigned int sequence; +}; +typedef struct ck_sequence ck_sequence_t; + +#define CK_SEQUENCE_INITIALIZER { .sequence = 0 } + +CK_CC_INLINE static void +ck_sequence_init(struct ck_sequence *sq) +{ + + ck_pr_store_uint(&sq->sequence, 0); + return; +} + +CK_CC_INLINE static unsigned int +ck_sequence_read_begin(const struct ck_sequence *sq) +{ + unsigned int version; + + for (;;) { + version = ck_pr_load_uint(&sq->sequence); + + /* + * If a sequence is even then associated data may be in a + * consistent state. + */ + if (CK_CC_LIKELY((version & 1) == 0)) + break; + + /* + * If a sequence is odd then a thread is in the middle of an + * update. Retry the read to avoid operating on inconsistent + * data. + */ + ck_pr_stall(); + } + + ck_pr_fence_load(); + return version; +} + +CK_CC_INLINE static bool +ck_sequence_read_retry(const struct ck_sequence *sq, unsigned int version) +{ + + /* + * If the sequence number was updated then a read should be + * re-attempted. + */ + ck_pr_fence_load(); + return ck_pr_load_uint(&sq->sequence) != version; +} + +#define CK_SEQUENCE_READ(seqlock, version) \ + for (*(version) = 1; \ + (*(version) != 0) && (*(version) = ck_sequence_read_begin(seqlock), 1); \ + *(version) = ck_sequence_read_retry(seqlock, *(version))) + +/* + * This must be called after a successful mutex acquisition. + */ +CK_CC_INLINE static void +ck_sequence_write_begin(struct ck_sequence *sq) +{ + + /* + * Increment the sequence to an odd number to indicate + * the beginning of a write update. + */ + ck_pr_store_uint(&sq->sequence, sq->sequence + 1); + ck_pr_fence_store(); + return; +} + +/* + * This must be called before mutex ownership is relinquished. + */ +CK_CC_INLINE static void +ck_sequence_write_end(struct ck_sequence *sq) +{ + + /* + * Increment the sequence to an even number to indicate + * completion of a write update. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&sq->sequence, sq->sequence + 1); + return; +} + +#endif /* CK_SEQUENCE_H */ diff --git a/include/ck_spinlock.h b/include/ck_spinlock.h new file mode 100644 index 0000000..e9585f2 --- /dev/null +++ b/include/ck_spinlock.h @@ -0,0 +1,61 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_H +#define CK_SPINLOCK_H + +#include "spinlock/anderson.h" +#include "spinlock/cas.h" +#include "spinlock/clh.h" +#include "spinlock/dec.h" +#include "spinlock/fas.h" +#include "spinlock/hclh.h" +#include "spinlock/mcs.h" +#include "spinlock/ticket.h" + +/* + * On tested x86, x86_64, PPC64 and SPARC64 targets, + * ck_spinlock_fas proved to have lowest latency + * in fast path testing or negligible degradation + * from faster but less robust implementations. + */ +#define CK_SPINLOCK_INITIALIZER CK_SPINLOCK_FAS_INITIALIZER +#define ck_spinlock_t ck_spinlock_fas_t +#define ck_spinlock_init(x) ck_spinlock_fas_init(x) +#define ck_spinlock_lock(x) ck_spinlock_fas_lock(x) +#define ck_spinlock_lock_eb(x) ck_spinlock_fas_lock_eb(x) +#define ck_spinlock_unlock(x) ck_spinlock_fas_unlock(x) +#define ck_spinlock_locked(x) ck_spinlock_fas_locked(x) +#define ck_spinlock_trylock(x) ck_spinlock_fas_trylock(x) + +CK_ELIDE_PROTOTYPE(ck_spinlock, ck_spinlock_t, + ck_spinlock_locked, ck_spinlock_lock, + ck_spinlock_locked, ck_spinlock_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock, ck_spinlock_t, + ck_spinlock_locked, ck_spinlock_trylock) + +#endif /* CK_SPINLOCK_H */ diff --git a/include/ck_stack.h b/include/ck_stack.h new file mode 100644 index 0000000..eb2b685 --- /dev/null +++ b/include/ck_stack.h @@ -0,0 +1,357 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_STACK_H +#define CK_STACK_H + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_stack_entry { + struct ck_stack_entry *next; +}; +typedef struct ck_stack_entry ck_stack_entry_t; + +struct ck_stack { + struct ck_stack_entry *head; + char *generation CK_CC_PACKED; +} CK_CC_ALIASED; +typedef struct ck_stack ck_stack_t; + +#define CK_STACK_INITIALIZER { NULL, NULL } + +#ifndef CK_F_STACK_PUSH_UPMC +#define CK_F_STACK_PUSH_UPMC +/* + * Stack producer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + while (ck_pr_cas_ptr_value(&target->head, stack, entry, &stack) == false) { + entry->next = stack; + ck_pr_fence_store(); + } + + return; +} +#endif /* CK_F_STACK_PUSH_UPMC */ + +#ifndef CK_F_STACK_TRYPUSH_UPMC +#define CK_F_STACK_TRYPUSH_UPMC +/* + * Stack producer operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. + */ +CK_CC_INLINE static bool +ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + return ck_pr_cas_ptr(&target->head, stack, entry); +} +#endif /* CK_F_STACK_TRYPUSH_UPMC */ + +#ifndef CK_F_STACK_POP_UPMC +#define CK_F_STACK_POP_UPMC +/* + * Stack consumer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry, *next; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return NULL; + + ck_pr_fence_load(); + next = entry->next; + while (ck_pr_cas_ptr_value(&target->head, entry, next, &entry) == false) { + if (entry == NULL) + break; + + ck_pr_fence_load(); + next = entry->next; + } + + return entry; +} +#endif + +#ifndef CK_F_STACK_TRYPOP_UPMC +#define CK_F_STACK_TRYPOP_UPMC +/* + * Stack production operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. The value pointed to by the second + * argument is set to a valid ck_stack_entry_t reference if true is returned. If + * false is returned, then the value pointed to by the second argument is undefined. + */ +CK_CC_INLINE static bool +ck_stack_trypop_upmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return false; + + ck_pr_fence_load(); + if (ck_pr_cas_ptr(&target->head, entry, entry->next) == true) { + *r = entry; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_UPMC */ + +#ifndef CK_F_STACK_BATCH_POP_UPMC +#define CK_F_STACK_BATCH_POP_UPMC +/* + * Pop all items off the stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_fas_ptr(&target->head, NULL); + ck_pr_fence_load(); + return entry; +} +#endif /* CK_F_STACK_BATCH_POP_UPMC */ + +#ifndef CK_F_STACK_PUSH_MPMC +#define CK_F_STACK_PUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + ck_stack_push_upmc(target, entry); + return; +} +#endif /* CK_F_STACK_PUSH_MPMC */ + +#ifndef CK_F_STACK_TRYPUSH_MPMC +#define CK_F_STACK_TRYPUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static bool +ck_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + return ck_stack_trypush_upmc(target, entry); +} +#endif /* CK_F_STACK_TRYPUSH_MPMC */ + +#ifdef CK_F_PR_CAS_PTR_2_VALUE +#ifndef CK_F_STACK_POP_MPMC +#define CK_F_STACK_POP_MPMC +/* + * Stack consumer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_mpmc(struct ck_stack *target) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return NULL; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + + update.generation = original.generation + 1; + update.head = original.head->next; + + while (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == false) { + if (original.head == NULL) + return NULL; + + update.generation = original.generation + 1; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + update.head = original.head->next; + } + + return original.head; +} +#endif /* CK_F_STACK_POP_MPMC */ + +#ifndef CK_F_STACK_TRYPOP_MPMC +#define CK_F_STACK_TRYPOP_MPMC +CK_CC_INLINE static bool +ck_stack_trypop_mpmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return false; + + update.generation = original.generation + 1; + ck_pr_fence_load(); + update.head = original.head->next; + + if (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == true) { + *r = original.head; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_MPMC */ +#endif /* CK_F_PR_CAS_PTR_2_VALUE */ + +#ifndef CK_F_STACK_BATCH_POP_MPMC +#define CK_F_STACK_BATCH_POP_MPMC +/* + * This is equivalent to the UP/MC version as NULL does not need a + * a generation count. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_mpmc(struct ck_stack *target) +{ + + return ck_stack_batch_pop_upmc(target); +} +#endif /* CK_F_STACK_BATCH_POP_MPMC */ + +#ifndef CK_F_STACK_PUSH_MPNC +#define CK_F_STACK_PUSH_MPNC +/* + * Stack producer operation safe with no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + entry->next = NULL; + ck_pr_fence_store_atomic(); + stack = ck_pr_fas_ptr(&target->head, entry); + ck_pr_store_ptr(&entry->next, stack); + ck_pr_fence_store(); + + return; +} +#endif /* CK_F_STACK_PUSH_MPNC */ + +/* + * Stack producer operation for single producer and no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_spnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + entry->next = target->head; + target->head = entry; + return; +} + +/* + * Stack consumer operation for no concurrent producers and single consumer. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + if (target->head == NULL) + return NULL; + + n = target->head; + target->head = n->next; + + return n; +} + +/* + * Pop all items off a stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + n = target->head; + target->head = NULL; + + return n; +} + +/* + * Stack initialization function. Guarantees initialization across processors. + */ +CK_CC_INLINE static void +ck_stack_init(struct ck_stack *stack) +{ + + stack->head = NULL; + stack->generation = NULL; + return; +} + +/* Defines a container_of functions for */ +#define CK_STACK_CONTAINER(T, M, N) CK_CC_CONTAINER(ck_stack_entry_t, T, M, N) + +#define CK_STACK_ISEMPTY(m) ((m)->head == NULL) +#define CK_STACK_FIRST(s) ((s)->head) +#define CK_STACK_NEXT(m) ((m)->next) +#define CK_STACK_FOREACH(stack, entry) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL; \ + (entry) = CK_STACK_NEXT(entry)) +#define CK_STACK_FOREACH_SAFE(stack, entry, T) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_STACK_H */ diff --git a/include/ck_stdbool.h b/include/ck_stdbool.h new file mode 100644 index 0000000..b9a7982 --- /dev/null +++ b/include/ck_stdbool.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/types.h> +#else +#include <stdbool.h> +#endif diff --git a/include/ck_stddef.h b/include/ck_stddef.h new file mode 100644 index 0000000..6019ea9 --- /dev/null +++ b/include/ck_stddef.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stddef.h> +#else +#include <stddef.h> +#endif diff --git a/include/ck_stdint.h b/include/ck_stdint.h new file mode 100644 index 0000000..8f416a9 --- /dev/null +++ b/include/ck_stdint.h @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include <linux/kernel.h> +#include <linux/types.h> +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/stdint.h> +#else +#include <stdint.h> +#endif /* __linux__ && __KERNEL__ */ diff --git a/include/ck_stdlib.h b/include/ck_stdlib.h new file mode 100644 index 0000000..c1ac69b --- /dev/null +++ b/include/ck_stdlib.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/systm.h> +#else +#include <stdlib.h> +#endif diff --git a/include/ck_string.h b/include/ck_string.h new file mode 100644 index 0000000..8d2c252 --- /dev/null +++ b/include/ck_string.h @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include <sys/systm.h> +#else +#include <string.h> +#endif diff --git a/include/ck_swlock.h b/include/ck_swlock.h new file mode 100644 index 0000000..ad6d3a0 --- /dev/null +++ b/include/ck_swlock.h @@ -0,0 +1,218 @@ +/* + * Copyright 2014 Jaidev Sridhar. + * Copyright 2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SWLOCK_H +#define CK_SWLOCK_H + +#include <ck_elide.h> +#include <ck_limits.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +struct ck_swlock { + uint32_t value; +}; +typedef struct ck_swlock ck_swlock_t; + +#define CK_SWLOCK_INITIALIZER {0} +#define CK_SWLOCK_WRITER_BIT (1UL << 31) +#define CK_SWLOCK_LATCH_BIT (1UL << 30) +#define CK_SWLOCK_WRITER_MASK (CK_SWLOCK_LATCH_BIT | CK_SWLOCK_WRITER_BIT) +#define CK_SWLOCK_READER_MASK (UINT32_MAX ^ CK_SWLOCK_WRITER_MASK) + +CK_CC_INLINE static void +ck_swlock_init(struct ck_swlock *rw) +{ + + rw->value = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_unlock(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked_writer(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_swlock_write_downgrade(ck_swlock_t *rw) +{ + + ck_pr_inc_32(&rw->value); + ck_swlock_write_unlock(rw); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_load_32(&rw->value); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static bool +ck_swlock_write_trylock(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT); + ck_pr_fence_lock(); + return r; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t, + ck_swlock_locked, ck_swlock_write_trylock) + +CK_CC_INLINE static void +ck_swlock_write_lock(ck_swlock_t *rw) +{ + + ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); + while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_latch(ck_swlock_t *rw) +{ + + /* Publish intent to acquire lock. */ + ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); + + /* Stall until readers have seen the writer and cleared. */ + while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT, + CK_SWLOCK_WRITER_MASK) == false) { + do { + ck_pr_stall(); + } while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_unlatch(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_store_32(&rw->value, 0); + return; +} + +CK_ELIDE_PROTOTYPE(ck_swlock_write, ck_swlock_t, + ck_swlock_locked, ck_swlock_write_lock, + ck_swlock_locked_writer, ck_swlock_write_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_read, ck_swlock_t, + ck_swlock_locked_writer, ck_swlock_read_trylock) + +CK_CC_INLINE static bool +ck_swlock_read_trylock(ck_swlock_t *rw) +{ + uint32_t l = ck_pr_load_32(&rw->value); + + if (l & CK_SWLOCK_WRITER_BIT) + return false; + + l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK; + if (l == CK_SWLOCK_WRITER_BIT) + ck_pr_dec_32(&rw->value); + + ck_pr_fence_lock(); + return l == 0; +} + +CK_CC_INLINE static void +ck_swlock_read_lock(ck_swlock_t *rw) +{ + uint32_t l; + + for (;;) { + while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT) + ck_pr_stall(); + + l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK; + if (l == 0) + break; + + /* + * If the latch bit has not been set, then the writer would + * have observed the reader and will wait to completion of + * read-side critical section. + */ + if (l == CK_SWLOCK_WRITER_BIT) + ck_pr_dec_32(&rw->value); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked_reader(ck_swlock_t *rw) +{ + + ck_pr_fence_load(); + return ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK; +} + +CK_CC_INLINE static void +ck_swlock_read_unlock(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_dec_32(&rw->value); + return; +} + +CK_ELIDE_PROTOTYPE(ck_swlock_read, ck_swlock_t, + ck_swlock_locked_writer, ck_swlock_read_lock, + ck_swlock_locked_reader, ck_swlock_read_unlock) + +#endif /* CK_SWLOCK_H */ diff --git a/include/ck_tflock.h b/include/ck_tflock.h new file mode 100644 index 0000000..a1872ae --- /dev/null +++ b/include/ck_tflock.h @@ -0,0 +1,136 @@ +/* + * Copyright 2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_TFLOCK_TICKET_H +#define CK_TFLOCK_TICKET_H + +/* + * This is an implementation of task-fair locks derived from the work + * described in: + * John M. Mellor-Crummey and Michael L. Scott. 1991. + * Scalable reader-writer synchronization for shared-memory + * multiprocessors. SIGPLAN Not. 26, 7 (April 1991), 106-113. + */ + +#include <ck_cc.h> +#include <ck_pr.h> + +struct ck_tflock_ticket { + uint32_t request; + uint32_t completion; +}; +typedef struct ck_tflock_ticket ck_tflock_ticket_t; + +#define CK_TFLOCK_TICKET_INITIALIZER { 0, 0 } + +#define CK_TFLOCK_TICKET_RC_INCR 0x10000U /* Read-side increment. */ +#define CK_TFLOCK_TICKET_WC_INCR 0x1U /* Write-side increment. */ +#define CK_TFLOCK_TICKET_W_MASK 0xffffU /* Write-side mask. */ +#define CK_TFLOCK_TICKET_WC_TOPMSK 0x8000U /* Write clear mask for overflow. */ +#define CK_TFLOCK_TICKET_RC_TOPMSK 0x80000000U /* Read clear mask for overflow. */ + +CK_CC_INLINE static uint32_t +ck_tflock_ticket_fca_32(uint32_t *target, uint32_t mask, uint32_t delta) +{ + uint32_t snapshot = ck_pr_load_32(target); + uint32_t goal; + + for (;;) { + goal = (snapshot & ~mask) + delta; + if (ck_pr_cas_32_value(target, snapshot, goal, &snapshot) == true) + break; + + ck_pr_stall(); + } + + return snapshot; +} + +CK_CC_INLINE static void +ck_tflock_ticket_init(struct ck_tflock_ticket *pf) +{ + + pf->request = pf->completion = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_write_lock(struct ck_tflock_ticket *lock) +{ + uint32_t previous; + + previous = ck_tflock_ticket_fca_32(&lock->request, CK_TFLOCK_TICKET_WC_TOPMSK, + CK_TFLOCK_TICKET_WC_INCR); + ck_pr_fence_atomic_load(); + while (ck_pr_load_32(&lock->completion) != previous) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_write_unlock(struct ck_tflock_ticket *lock) +{ + + ck_pr_fence_unlock(); + ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_WC_TOPMSK, + CK_TFLOCK_TICKET_WC_INCR); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_read_lock(struct ck_tflock_ticket *lock) +{ + uint32_t previous; + + previous = ck_tflock_ticket_fca_32(&lock->request, + CK_TFLOCK_TICKET_RC_TOPMSK, CK_TFLOCK_TICKET_RC_INCR) & + CK_TFLOCK_TICKET_W_MASK; + + ck_pr_fence_atomic_load(); + + while ((ck_pr_load_32(&lock->completion) & + CK_TFLOCK_TICKET_W_MASK) != previous) { + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_read_unlock(struct ck_tflock_ticket *lock) +{ + + ck_pr_fence_unlock(); + ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_RC_TOPMSK, + CK_TFLOCK_TICKET_RC_INCR); + return; +} + +#endif /* CK_TFLOCK_TICKET_H */ diff --git a/include/gcc/aarch64/ck_f_pr.h b/include/gcc/aarch64/ck_f_pr.h new file mode 100644 index 0000000..93ecee0 --- /dev/null +++ b/include/gcc/aarch64/ck_f_pr.h @@ -0,0 +1,167 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT diff --git a/include/gcc/aarch64/ck_pr.h b/include/gcc/aarch64/ck_pr.h new file mode 100644 index 0000000..e739c4d --- /dev/null +++ b/include/gcc/aarch64/ck_pr.h @@ -0,0 +1,227 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_H +#define CK_PR_AARCH64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#define CK_DMB_SY __asm __volatile("dmb ish" : : "r" (0) : "memory") +#define CK_DMB_LD __asm __volatile("dmb ishld" : : "r" (0) : "memory") +#define CK_DMB_ST __asm __volatile("dmb ishst" : : "r" (0) : "memory") + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_SY) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB_SY) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB_SY) +CK_PR_FENCE(load, CK_DMB_LD) +CK_PR_FENCE(load_store, CK_DMB_SY) +CK_PR_FENCE(memory, CK_DMB_SY) +CK_PR_FENCE(acquire, CK_DMB_SY) +CK_PR_FENCE(release, CK_DMB_SY) +CK_PR_FENCE(acqrel, CK_DMB_SY) +CK_PR_FENCE(lock, CK_DMB_SY) +CK_PR_FENCE(unlock, CK_DMB_SY) + +#undef CK_PR_FENCE + +#undef CK_DMB_SI +#undef CK_DMB_LD +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %w0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } +#define CK_PR_LOAD_64(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + + +CK_PR_LOAD_64(ptr, void, void *, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, I) +#define CK_PR_LOAD_S_64(S, T, I) CK_PR_LOAD_64(S, T, T, I) + +CK_PR_LOAD_S_64(64, uint64_t, "ldr") +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S_64(double, double, "ldr") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD_S_64 +#undef CK_PR_LOAD +#undef CK_PR_LAOD_64 + +#define CK_PR_STORE(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %w1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } +#define CK_PR_STORE_64(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_64(ptr, void, const void *, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, I) +#define CK_PR_STORE_S_64(S, T, I) CK_PR_STORE_64(S, T, T, I) + +CK_PR_STORE_S_64(64, uint64_t, "str") +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S_64(double, double, "str") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_S_64 +#undef CK_PR_STORE +#undef CK_PR_STORE_64 + +#ifdef CK_MD_LSE_ENABLE +#include "ck_pr_lse.h" +#else +#include "ck_pr_llsc.h" +#endif + +/* + * ck_pr_neg_*() functions can only be implemented via LL/SC, as there are no + * LSE alternatives. + */ +#define CK_PR_NEG(N, M, T, W, R) \ + CK_CC_INLINE static void \ + ck_pr_neg_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "neg %" R "0, %" R "0;" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_NEG(ptr, void, void *, "", "") +CK_PR_NEG(64, uint64_t, uint64_t, "", "") + +#define CK_PR_NEG_S(S, T, W) \ + CK_PR_NEG(S, T, T, W, "w") \ + +CK_PR_NEG_S(32, uint32_t, "") +CK_PR_NEG_S(uint, unsigned int, "") +CK_PR_NEG_S(int, int, "") +CK_PR_NEG_S(16, uint16_t, "h") +CK_PR_NEG_S(8, uint8_t, "b") +CK_PR_NEG_S(short, short, "h") +CK_PR_NEG_S(char, char, "b") + +#undef CK_PR_NEG_S +#undef CK_PR_NEG + +#endif /* CK_PR_AARCH64_H */ + diff --git a/include/gcc/aarch64/ck_pr_llsc.h b/include/gcc/aarch64/ck_pr_llsc.h new file mode 100644 index 0000000..aa4e309 --- /dev/null +++ b/include/gcc/aarch64/ck_pr_llsc.h @@ -0,0 +1,352 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_LLSC_H +#define CK_PR_AARCH64_LLSC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) +{ + uint64_t tmp1, tmp2; + + __asm__ __volatile__("1:" + "ldxp %0, %1, [%4];" + "mov %2, %0;" + "mov %3, %1;" + "eor %0, %0, %5;" + "eor %1, %1, %6;" + "orr %1, %0, %1;" + "mov %w0, #0;" + "cbnz %1, 2f;" + "stxp %w0, %7, %8, [%4];" + "cbnz %w0, 1b;" + "mov %w0, #1;" + "2:" + : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1]) + : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) + : "cc", "memory"); + + return (tmp1); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set), + CK_CPP_CAST(uint64_t *, value))); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + uint64_t tmp1, tmp2; + + __asm__ __volatile__("1:" + "ldxp %0, %1, [%2];" + "eor %0, %0, %3;" + "eor %1, %1, %4;" + "orr %1, %0, %1;" + "mov %w0, #0;" + "cbnz %1, 2f;" + "stxp %w0, %5, %6, [%2];" + "cbnz %w0, 1b;" + "mov %w0, #1;" + "2:" + : "=&r" (tmp1), "=&r" (tmp2) + : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) + : "cc", "memory"); + + return (tmp1); +} +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set))); +} + + +#define CK_PR_CAS(N, M, T, W, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "cmp %" R "0, %" R "4;" \ + "b.ne 2f;" \ + "stxr" W " %w1, %" R "3, [%2];" \ + "cbnz %w1, 1b;" \ + "2:" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *(T *)value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__( \ + "1:" \ + "ldxr" W " %" R "0, [%2];" \ + "cmp %" R "0, %" R "4;" \ + "b.ne 2f;" \ + "stxr" W " %w1, %" R "3, [%2];" \ + "cbnz %w1, 1b;" \ + "2:" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void, void *, "", "") + +#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) +CK_PR_CAS_S(64, uint64_t, "", "") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "", "") +#endif +CK_PR_CAS_S(32, uint32_t, "", "w") +CK_PR_CAS_S(uint, unsigned int, "", "w") +CK_PR_CAS_S(int, int, "", "w") +CK_PR_CAS_S(16, uint16_t, "h", "w") +CK_PR_CAS_S(8, uint8_t, "b", "w") +CK_PR_CAS_S(short, short, "h", "w") +CK_PR_CAS_S(char, char, "b", "w") + + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W, R) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "stxr" W " %w1, %" R "3, [%2];"\ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "", "") +CK_PR_FAS(32, uint32_t, uint32_t, "", "w") +CK_PR_FAS(ptr, void, void *, "", "") +CK_PR_FAS(int, int, int, "", "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "", "w") +CK_PR_FAS(16, uint16_t, uint16_t, "h", "w") +CK_PR_FAS(8, uint8_t, uint8_t, "b", "w") +CK_PR_FAS(short, short, short, "h", "w") +CK_PR_FAS(char, char, char, "b", "w") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W, R) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + I ";" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "", "") +CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "", "") +CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "", "") +CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "add %0, %0, #1", "", "") +CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "sub %0, %0, #1", "", "") +CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mvn %0, %0", "", "") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "add %w0, %w0, #1", W, "w") \ + CK_PR_UNARY(dec, S, T, T, "sub %w0, %w0, #1", W, "w") \ + CK_PR_UNARY(not, S, T, T, "mvn %w0, %w0", W, "w") \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W, R) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];"\ + I " %" R "0, %" R "0, %" R "3;" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "", "") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "", "") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "") +CK_PR_BINARY(and, 64, uint64_t, uint64_t, "and", "", "") +CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "") +CK_PR_BINARY(or, 64, uint64_t, uint64_t, "orr", "", "") +CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "sub", "", "") +CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W, "w") \ + CK_PR_BINARY(add, S, T, T, "add", W, "w") \ + CK_PR_BINARY(or, S, T, T, "orr", W, "w") \ + CK_PR_BINARY(sub, S, T, T, "sub", W, "w") \ + CK_PR_BINARY(xor, S, T, T, "eor", W, "w") + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldxr %0, [%3];" + "add %1, %4, %0;" + "stxr %w2, %1, [%3];" + "cbnz %w2, 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +CK_CC_INLINE static uint64_t +ck_pr_faa_64(uint64_t *target, uint64_t delta) +{ + uint64_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldxr %0, [%3];" + "add %1, %4, %0;" + "stxr %w2, %1, [%3];" + "cbnz %w2, 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r, tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %w0, [%3];" \ + "add %w1, %w4, %w0;" \ + "stxr" W " %w2, %w1, [%3];" \ + "cbnz %w2, 1b;" \ + : "=&r" (previous), \ + "=&r" (r), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_AARCH64_LLSC_H */ diff --git a/include/gcc/aarch64/ck_pr_lse.h b/include/gcc/aarch64/ck_pr_lse.h new file mode 100644 index 0000000..e2c9554 --- /dev/null +++ b/include/gcc/aarch64/ck_pr_lse.h @@ -0,0 +1,298 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * Copyright 2016 Alexey Kopytov. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_LSE_H +#define CK_PR_AARCH64_LSE_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) +{ + uint64_t tmp1; + uint64_t tmp2; + register uint64_t x0 __asm__ ("x0") = compare[0]; + register uint64_t x1 __asm__ ("x1") = compare[1]; + register uint64_t x2 __asm__ ("x2") = set[0]; + register uint64_t x3 __asm__ ("x3") = set[1]; + + __asm__ __volatile__("casp %0, %1, %4, %5, [%6];" + "eor %2, %0, %7;" + "eor %3, %1, %8;" + "orr %2, %2, %3;" + : "+&r" (x0), "+&r" (x1), "=&r" (tmp1), "=&r" (tmp2) + : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) + : "memory"); + + value[0] = x0; + value[1] = x1; + + return (!!tmp1); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set), + CK_CPP_CAST(uint64_t *, value))); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + register uint64_t x0 __asm__ ("x0") = compare[0]; + register uint64_t x1 __asm__ ("x1") = compare[1]; + register uint64_t x2 __asm__ ("x2") = set[0]; + register uint64_t x3 __asm__ ("x3") = set[1]; + + __asm__ __volatile__("casp %0, %1, %2, %3, [%4];" + "eor %0, %0, %5;" + "eor %1, %1, %6;" + "orr %0, %0, %1;" + : "+&r" (x0), "+&r" (x1) + : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) + : "memory"); + + return (!!x0); +} +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set))); +} + + +#define CK_PR_CAS(N, M, T, W, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + *(T *)value = compare; \ + __asm__ __volatile__( \ + "cas" W " %" R "0, %" R "2, [%1];" \ + : "+&r" (*(T *)value) \ + : "r" (target), \ + "r" (set) \ + : "memory"); \ + return (*(T *)value == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous = compare; \ + __asm__ __volatile__( \ + "cas" W " %" R "0, %" R "2, [%1];" \ + : "+&r" (previous) \ + : "r" (target), \ + "r" (set) \ + : "memory"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void, void *, "", "") + +#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) +CK_PR_CAS_S(64, uint64_t, "", "") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "", "") +#endif +CK_PR_CAS_S(32, uint32_t, "", "w") +CK_PR_CAS_S(uint, unsigned int, "", "w") +CK_PR_CAS_S(int, int, "", "w") +CK_PR_CAS_S(16, uint16_t, "h", "w") +CK_PR_CAS_S(8, uint8_t, "b", "w") +CK_PR_CAS_S(short, short, "h", "w") +CK_PR_CAS_S(char, char, "b", "w") + + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W, R) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__( \ + "swp" W " %" R "2, %" R "0, [%1];" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "", "") +CK_PR_FAS(32, uint32_t, uint32_t, "", "w") +CK_PR_FAS(ptr, void, void *, "", "") +CK_PR_FAS(int, int, int, "", "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "", "w") +CK_PR_FAS(16, uint16_t, uint16_t, "h", "w") +CK_PR_FAS(8, uint8_t, uint8_t, "b", "w") +CK_PR_FAS(short, short, short, "h", "w") +CK_PR_FAS(char, char, char, "b", "w") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W, R, S) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + __asm__ __volatile__(I ";" \ + "st" S W " " R "0, [%0];" \ + : \ + : "r" (target) \ + : "x0", "memory"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "mov x0, 1", "", "x", "add") +CK_PR_UNARY(dec, ptr, void, void *, "mov x0, -1", "", "x", "add") +CK_PR_UNARY(not, ptr, void, void *, "mov x0, -1", "", "x", "eor") +CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "mov x0, 1", "", "x", "add") +CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "add") +CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "eor") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "mov w0, 1", W, "w", "add") \ + CK_PR_UNARY(dec, S, T, T, "mov w0, -1", W, "w", "add") \ + CK_PR_UNARY(not, S, T, T, "mov w0, -1", W, "w", "eor") \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, S, W, R, I) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + __asm__ __volatile__(I ";" \ + "st" S W " %" R "0, [%1];" \ + : "+&r" (delta) \ + : "r" (target) \ + : "memory"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "clr", "", "", "mvn %0, %0") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "set", "", "", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "add", "", "", "neg %0, %0") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "", "") +CK_PR_BINARY(and, 64, uint64_t, uint64_t, "clr", "", "", "mvn %0, %0") +CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "", "") +CK_PR_BINARY(or, 64, uint64_t, uint64_t, "set", "", "", "") +CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "add", "", "", "neg %0, %0") +CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "clr", W, "w", "mvn %w0, %w0") \ + CK_PR_BINARY(add, S, T, T, "add", W, "w", "") \ + CK_PR_BINARY(or, S, T, T, "set", W, "w", "") \ + CK_PR_BINARY(sub, S, T, T, "add", W, "w", "neg %w0, %w0") \ + CK_PR_BINARY(xor, S, T, T, "eor", W, "w", "") + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous; + + __asm__ __volatile__( + "ldadd %2, %0, [%1];" + : "=r" (previous) + : "r" (target), + "r" (delta) + : "memory"); + + return (void *)(previous); +} + +CK_CC_INLINE static uint64_t +ck_pr_faa_64(uint64_t *target, uint64_t delta) +{ + uint64_t previous; + + __asm__ __volatile__( + "ldadd %2, %0, [%1];" + : "=r" (previous) + : "r" (target), + "r" (delta) + : "memory"); + + return (previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__( \ + "ldadd" W " %w2, %w0, [%1];" \ + : "=r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_AARCH64_LSE_H */ diff --git a/include/gcc/arm/ck_f_pr.h b/include/gcc/arm/ck_f_pr.h new file mode 100644 index 0000000..c508f85 --- /dev/null +++ b/include/gcc/arm/ck_f_pr.h @@ -0,0 +1,162 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#endif +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#endif +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#endif +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#endif +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT diff --git a/include/gcc/arm/ck_pr.h b/include/gcc/arm/ck_pr.h new file mode 100644 index 0000000..841ca21 --- /dev/null +++ b/include/gcc/arm/ck_pr.h @@ -0,0 +1,563 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2013-2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_ARM_H +#define CK_PR_ARM_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory") +#define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory") +#define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory") +/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */ +#ifdef __FreeBSD__ +#define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory") +#else +#define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory") +#endif /* __FreeBSD__ */ +#else +/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */ +#define CK_ISB \ + __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") +#define CK_DSB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") +#define CK_DMB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") +#define CK_DMB_ST CK_DMB +#endif + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_ST) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB) +CK_PR_FENCE(load, CK_DMB) +CK_PR_FENCE(load_store, CK_DMB) +CK_PR_FENCE(memory, CK_DMB) +CK_PR_FENCE(acquire, CK_DMB) +CK_PR_FENCE(release, CK_DMB) +CK_PR_FENCE(acqrel, CK_DMB) +CK_PR_FENCE(lock, CK_DMB) +CK_PR_FENCE(unlock, CK_DMB) + +#undef CK_PR_FENCE + +#undef CK_ISB +#undef CK_DSB +#undef CK_DMB +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_LOAD(T, N) \ +CK_CC_INLINE static T \ +ck_pr_md_load_##N(const T *target) \ +{ \ + register T ret; \ + \ + __asm __volatile("ldrexd %0, [%1]" \ + : "=&r" (ret) \ + : "r" (target) \ + : "memory", "cc"); \ + return (ret); \ +} + +CK_PR_DOUBLE_LOAD(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_LOAD(double, double) +#endif +#undef CK_PR_DOUBLE_LOAD +#endif + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_STORE(T, N) \ +CK_CC_INLINE static void \ +ck_pr_md_store_##N(const T *target, T value) \ +{ \ + T tmp; \ + uint32_t flag; \ + __asm __volatile("1: \n" \ + "ldrexd %0, [%2]\n" \ + "strexd %1, %3, [%2]\n" \ + "teq %1, #0\n" \ + "it ne \n" \ + "bne 1b\n" \ + : "=&r" (tmp), "=&r" (flag) \ + : "r" (target), "r" (value) \ + : "memory", "cc"); \ +} + +CK_PR_DOUBLE_STORE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_STORE(double, double) +#endif + +#undef CK_PR_DOUBLE_STORE + +#define CK_PR_DOUBLE_CAS_VALUE(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ +{ \ + T previous; \ + int tmp; \ + \ + __asm__ __volatile__("1:" \ + "ldrexd %0, [%4];" \ + "cmp %Q0, %Q2;" \ + "ittt eq;" \ + "cmpeq %R0, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + :"=&r" (previous), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + *value = previous; \ + return (*value == compare); \ +} + +CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS_VALUE(double, double) +#endif + +#undef CK_PR_DOUBLE_CAS_VALUE + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + + return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target), + __compare, + __set, + CK_CPP_CAST(uint64_t *, value))); +} + +#define CK_PR_DOUBLE_CAS(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N(T *target, T compare, T set) \ +{ \ + int ret; \ + T tmp; \ + \ + __asm__ __volatile__("1:" \ + "mov %0, #0;" \ + "ldrexd %1, [%4];" \ + "cmp %Q1, %Q2;" \ + "itttt eq;" \ + "cmpeq %R1, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "moveq %0, #1;" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "=&r" (ret), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + \ + return (ret); \ +} + +CK_PR_DOUBLE_CAS(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS(double, double) +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target), + __compare, + __set)); +} + +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + *(void **)value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + return (previous == compare); +} + +#define CK_PR_CAS(N, T, W) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + /* \ + * Using "+&" instead of "=&" to avoid bogus \ + * clang warnings. \ + */ \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t, "") +CK_PR_CAS(uint, unsigned int, "") +CK_PR_CAS(int, int, "") +CK_PR_CAS(16, uint16_t, "h") +CK_PR_CAS(8, uint8_t, "b") +CK_PR_CAS(short, short, "h") +CK_PR_CAS(char, char, "b") + + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "strex" W " %1, %3, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "") +CK_PR_FAS(ptr, void, void *, "") +CK_PR_FAS(int, int, int, "") +CK_PR_FAS(uint, unsigned int, unsigned int, "") +CK_PR_FAS(16, uint16_t, uint16_t, "h") +CK_PR_FAS(8, uint8_t, uint8_t, "b") +CK_PR_FAS(short, short, short, "h") +CK_PR_FAS(char, char, char, "b") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I ";" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "") +CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "") +CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \ + CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \ + CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I " %0, %0, %3;" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "orr", W) \ + CK_PR_BINARY(sub, S, T, T, "sub", W) \ + CK_PR_BINARY(xor, S, T, T, "eor", W) + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldrex %0, [%3];" + "add %1, %4, %0;" + "strex %2, %1, [%3];" + "cmp %2, #0;" + "bne 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous = 0, r = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%3];" \ + "add %1, %4, %0;" \ + "strex" W " %2, %1, [%3];" \ + "cmp %2, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (r), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_ARM_H */ + diff --git a/include/gcc/ck_cc.h b/include/gcc/ck_cc.h new file mode 100644 index 0000000..a14a4b5 --- /dev/null +++ b/include/gcc/ck_cc.h @@ -0,0 +1,142 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_GCC_CC_H +#define CK_GCC_CC_H + +#include <ck_md.h> + +#ifdef __SUNPRO_C +#define CK_CC_UNUSED +#define CK_CC_USED +#define CK_CC_IMM +#define CK_CC_IMM_U32 +#else +#define CK_CC_UNUSED __attribute__((unused)) +#define CK_CC_USED __attribute__((used)) +#define CK_CC_IMM "i" +#if defined(__x86_64__) || defined(__x86__) +#define CK_CC_IMM_U32 "Z" +#define CK_CC_IMM_S32 "e" +#else +#define CK_CC_IMM_U32 CK_CC_IMM +#define CK_CC_IMM_S32 CK_CC_IMM +#endif /* __x86_64__ || __x86__ */ +#endif + +#ifdef __OPTIMIZE__ +#define CK_CC_INLINE CK_CC_UNUSED inline +#else +#define CK_CC_INLINE CK_CC_UNUSED +#endif + +#define CK_CC_FORCE_INLINE CK_CC_UNUSED __attribute__((always_inline)) inline +#define CK_CC_RESTRICT __restrict__ + +/* + * Packed attribute. + */ +#define CK_CC_PACKED __attribute__((packed)) + +/* + * Weak reference. + */ +#define CK_CC_WEAKREF __attribute__((weakref)) + +/* + * Alignment attribute. + */ +#define CK_CC_ALIGN(B) __attribute__((aligned(B))) + +/* + * Cache align. + */ +#define CK_CC_CACHELINE CK_CC_ALIGN(CK_MD_CACHELINE) + +/* + * These are functions which should be avoided. + */ +#ifdef __freestanding__ +#pragma GCC poison malloc free +#endif + +/* + * Branch execution hints. + */ +#define CK_CC_LIKELY(x) (__builtin_expect(!!(x), 1)) +#define CK_CC_UNLIKELY(x) (__builtin_expect(!!(x), 0)) + +/* + * Some compilers are overly strict regarding aliasing semantics. + * Unfortunately, in many cases it makes more sense to pay aliasing + * cost rather than overly expensive register spillage. + */ +#define CK_CC_ALIASED __attribute__((__may_alias__)) + +/* + * Compile-time typeof + */ +#define CK_CC_TYPEOF(X, DEFAULT) __typeof__(X) + +/* + * Portability wrappers for bitwise ops. + */ + +#define CK_F_CC_FFS +#define CK_F_CC_CLZ +#define CK_F_CC_CTZ +#define CK_F_CC_POPCOUNT + +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + + return __builtin_ffs(x); +} + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + + return __builtin_clz(x); +} + +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + + return __builtin_ctz(x); +} + +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + + return __builtin_popcount(x); +} + +#endif /* CK_GCC_CC_H */ diff --git a/include/gcc/ck_f_pr.h b/include/gcc/ck_f_pr.h new file mode 100644 index 0000000..0ef0d10 --- /dev/null +++ b/include/gcc/ck_f_pr.h @@ -0,0 +1,105 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT diff --git a/include/gcc/ck_pr.h b/include/gcc/ck_pr.h new file mode 100644 index 0000000..084d423 --- /dev/null +++ b/include/gcc/ck_pr.h @@ -0,0 +1,297 @@ +/* + * Copyright 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_GCC_H +#define CK_PR_GCC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> + +CK_CC_INLINE static void +ck_pr_barrier(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#ifndef CK_F_PR +#define CK_F_PR + +#include <ck_stdbool.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +#define CK_PR_ACCESS(x) (*(volatile __typeof__(x) *)&(x)) + +#define CK_PR_LOAD(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + ck_pr_barrier(); \ + r = CK_PR_ACCESS(*(const T *)target); \ + ck_pr_barrier(); \ + return (r); \ + } \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + ck_pr_barrier(); \ + CK_PR_ACCESS(*(T *)target) = v; \ + ck_pr_barrier(); \ + return; \ + } + +CK_CC_INLINE static void * +ck_pr_md_load_ptr(const void *target) +{ + void *r; + + ck_pr_barrier(); + r = CK_CC_DECONST_PTR(CK_PR_ACCESS(target)); + ck_pr_barrier(); + + return r; +} + +CK_CC_INLINE static void +ck_pr_md_store_ptr(void *target, const void *v) +{ + + ck_pr_barrier(); + CK_PR_ACCESS(target) = CK_CC_DECONST_PTR(v); + ck_pr_barrier(); + return; +} + +#define CK_PR_LOAD_S(S, T) CK_PR_LOAD(S, T, T) + +CK_PR_LOAD_S(char, char) +CK_PR_LOAD_S(uint, unsigned int) +CK_PR_LOAD_S(int, int) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double) +#endif +CK_PR_LOAD_S(64, uint64_t) +CK_PR_LOAD_S(32, uint32_t) +CK_PR_LOAD_S(16, uint16_t) +CK_PR_LOAD_S(8, uint8_t) + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + ck_pr_barrier(); +} + +/* + * Load and store fences are equivalent to full fences in the GCC port. + */ +#define CK_PR_FENCE(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __sync_synchronize(); \ + } + +CK_PR_FENCE(atomic) +CK_PR_FENCE(atomic_atomic) +CK_PR_FENCE(atomic_load) +CK_PR_FENCE(atomic_store) +CK_PR_FENCE(store_atomic) +CK_PR_FENCE(load_atomic) +CK_PR_FENCE(load) +CK_PR_FENCE(load_load) +CK_PR_FENCE(load_store) +CK_PR_FENCE(store) +CK_PR_FENCE(store_store) +CK_PR_FENCE(store_load) +CK_PR_FENCE(memory) +CK_PR_FENCE(acquire) +CK_PR_FENCE(release) +CK_PR_FENCE(acqrel) +CK_PR_FENCE(lock) +CK_PR_FENCE(unlock) + +#undef CK_PR_FENCE + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + z = __sync_bool_compare_and_swap((T *)target, compare, set); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *) + +#define CK_PR_CAS_S(S, T) CK_PR_CAS(S, T, T) + +CK_PR_CAS_S(char, char) +CK_PR_CAS_S(int, int) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(64, uint64_t) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(16, uint16_t) +CK_PR_CAS_S(8, uint8_t) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *v) +{ + set = __sync_val_compare_and_swap((void **)target, compare, set); + *(void **)v = set; + return (set == compare); +} + +#define CK_PR_CAS_O(S, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \ + { \ + set = __sync_val_compare_and_swap(target, compare, set);\ + *v = set; \ + return (set == compare); \ + } + +CK_PR_CAS_O(char, char) +CK_PR_CAS_O(int, int) +CK_PR_CAS_O(uint, unsigned int) +CK_PR_CAS_O(64, uint64_t) +CK_PR_CAS_O(32, uint32_t) +CK_PR_CAS_O(16, uint16_t) +CK_PR_CAS_O(8, uint8_t) + +#undef CK_PR_CAS_O + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_add((T *)target, d); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, void *) + +#define CK_PR_FAA_S(S, T) CK_PR_FAA(S, T, T) + +CK_PR_FAA_S(char, char) +CK_PR_FAA_S(uint, unsigned int) +CK_PR_FAA_S(int, int) +CK_PR_FAA_S(64, uint64_t) +CK_PR_FAA_S(32, uint32_t) +CK_PR_FAA_S(16, uint16_t) +CK_PR_FAA_S(8, uint8_t) + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_##K((T *)target, d); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T) CK_PR_BINARY(K, S, T, T) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, void *) \ + CK_PR_BINARY_S(K, char, char) \ + CK_PR_BINARY_S(K, int, int) \ + CK_PR_BINARY_S(K, uint, unsigned int) \ + CK_PR_BINARY_S(K, 64, uint64_t) \ + CK_PR_BINARY_S(K, 32, uint32_t) \ + CK_PR_BINARY_S(K, 16, uint16_t) \ + CK_PR_BINARY_S(K, 8, uint8_t) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +#define CK_PR_UNARY(S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_inc_##S(M *target) \ + { \ + ck_pr_add_##S(target, (T)1); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_pr_dec_##S(M *target) \ + { \ + ck_pr_sub_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_S(S, M) CK_PR_UNARY(S, M, M) + +CK_PR_UNARY(ptr, void, void *) +CK_PR_UNARY_S(char, char) +CK_PR_UNARY_S(int, int) +CK_PR_UNARY_S(uint, unsigned int) +CK_PR_UNARY_S(64, uint64_t) +CK_PR_UNARY_S(32, uint32_t) +CK_PR_UNARY_S(16, uint16_t) +CK_PR_UNARY_S(8, uint8_t) + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY +#endif /* !CK_F_PR */ +#endif /* CK_PR_GCC_H */ diff --git a/include/gcc/ppc/ck_f_pr.h b/include/gcc/ppc/ck_f_pr.h new file mode 100644 index 0000000..0aec33e --- /dev/null +++ b/include/gcc/ppc/ck_f_pr.h @@ -0,0 +1,79 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/include/gcc/ppc/ck_pr.h b/include/gcc/ppc/ck_pr.h new file mode 100644 index 0000000..cd7935d --- /dev/null +++ b/include/gcc/ppc/ck_pr.h @@ -0,0 +1,327 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2012 João Fernandes. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC_H +#define CK_PR_PPC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "lwz") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "stw") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#define CK_PR_CAS(N, T, M) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *(T *)value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void *, void) +#define CK_PR_CAS_S(a, b) CK_PR_CAS(a, b, b) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(int, int) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "w") +CK_PR_FAS(ptr, void, void *, "w") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "w") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "w") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "w") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "w") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "w") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "w") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "w") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "w") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "w") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "lwarx %0, 0, %2;" + "add %1, %3, %0;" + "stwcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC_H */ + diff --git a/include/gcc/ppc64/ck_f_pr.h b/include/gcc/ppc64/ck_f_pr.h new file mode 100644 index 0000000..cd54a28 --- /dev/null +++ b/include/gcc/ppc64/ck_f_pr.h @@ -0,0 +1,97 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/include/gcc/ppc64/ck_pr.h b/include/gcc/ppc64/ck_pr.h new file mode 100644 index 0000000..3f5e5db --- /dev/null +++ b/include/gcc/ppc64/ck_pr.h @@ -0,0 +1,427 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC64_H +#define CK_PR_PPC64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * These are derived from: + * http://www.ibm.com/developerworks/systems/articles/powerpc.html + */ +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ld") +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "ld") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "std") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(64, uint64_t, "std") +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S(double, double, "std") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + *value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + ck_pr_md_store_ptr(value, previous); + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "d") +CK_PR_FAS(32, uint32_t, uint32_t, "w") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS(double, double, double, "d") +#endif +CK_PR_FAS(ptr, void, void *, "d") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(64, uint64_t, "d") +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(64, uint64_t, "d") +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %2;" + "add %1, %3, %0;" + "stdcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(64, uint64_t, "d") +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC64_H */ diff --git a/include/gcc/sparcv9/ck_f_pr.h b/include/gcc/sparcv9/ck_f_pr.h new file mode 100644 index 0000000..0398680 --- /dev/null +++ b/include/gcc/sparcv9/ck_f_pr.h @@ -0,0 +1,26 @@ +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_INT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_UINT +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR + diff --git a/include/gcc/sparcv9/ck_pr.h b/include/gcc/sparcv9/ck_pr.h new file mode 100644 index 0000000..767af6a --- /dev/null +++ b/include/gcc/sparcv9/ck_pr.h @@ -0,0 +1,228 @@ +/* + * Copyright 2009, 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_SPARCV9_H +#define CK_PR_SPARCV9_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * Order loads at the least. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("membar #LoadLoad" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * Atomic operations are treated as both load and store + * operations on SPARCv9. + */ +CK_PR_FENCE(atomic, "membar #StoreStore") +CK_PR_FENCE(atomic_store, "membar #StoreStore") +CK_PR_FENCE(atomic_load, "membar #StoreLoad") +CK_PR_FENCE(store_atomic, "membar #StoreStore") +CK_PR_FENCE(load_atomic, "membar #LoadStore") +CK_PR_FENCE(store, "membar #StoreStore") +CK_PR_FENCE(store_load, "membar #StoreLoad") +CK_PR_FENCE(load, "membar #LoadLoad") +CK_PR_FENCE(load_store, "membar #LoadStore") +CK_PR_FENCE(memory, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad") +CK_PR_FENCE(acquire, "membar #LoadLoad | #LoadStore") +CK_PR_FENCE(release, "membar #LoadStore | #StoreStore") +CK_PR_FENCE(acqrel, "membar #LoadLoad | #LoadStore | #StoreStore") +CK_PR_FENCE(lock, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad") +CK_PR_FENCE(unlock, "membar #LoadStore | #StoreStore") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " [%1], %0" \ + : "=&r" (r) \ + : "r" (target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ldx") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ldx") +CK_PR_LOAD_S(32, uint32_t, "lduw") +CK_PR_LOAD_S(uint, unsigned int, "lduw") +CK_PR_LOAD_S(double, double, "ldx") +CK_PR_LOAD_S(int, int, "ldsw") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, [%1]" \ + : \ + : "r" (v), \ + "r" (target) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "stx") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(8, uint8_t, "stub") +CK_PR_STORE_S(64, uint64_t, "stx") +CK_PR_STORE_S(32, uint32_t, "stuw") +CK_PR_STORE_S(uint, unsigned int, "stuw") +CK_PR_STORE_S(double, double, "stx") +CK_PR_STORE_S(int, int, "stsw") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + *value = set; + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + + return ck_pr_cas_64(target, (uint64_t)compare, (uint64_t)set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *previous) +{ + + return ck_pr_cas_64_value(target, (uint64_t)compare, (uint64_t)set, previous); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + *value = set; \ + return (compare == set); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + return (compare == set); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, T) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(T *target, T update) \ + { \ + \ + __asm__ __volatile__("swap [%1], %0" \ + : "+&r" (update) \ + : "r" (target) \ + : "memory"); \ + return (update); \ + } + +CK_PR_FAS(int, int) +CK_PR_FAS(uint, unsigned int) +CK_PR_FAS(32, uint32_t) + +#undef CK_PR_FAS + +#endif /* CK_PR_SPARCV9_H */ + diff --git a/include/gcc/x86/ck_f_pr.h b/include/gcc/x86/ck_f_pr.h new file mode 100644 index 0000000..f82c66b --- /dev/null +++ b/include/gcc/x86/ck_f_pr.h @@ -0,0 +1,152 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/include/gcc/x86/ck_pr.h b/include/gcc/x86/ck_pr.h new file mode 100644 index 0000000..a04cebf --- /dev/null +++ b/include/gcc/x86/ck_pr.h @@ -0,0 +1,390 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 Devon H. O'Dell <devon.odell@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_H +#define CK_PR_X86_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#else +#define CK_PR_LOCK_PREFIX "lock " +#endif + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) + * or "predefined delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "sfence") +CK_PR_FENCE(atomic_store, "sfence") +CK_PR_FENCE(atomic_load, "mfence") +CK_PR_FENCE(store_atomic, "sfence") +CK_PR_FENCE(load_atomic, "mfence") +CK_PR_FENCE(load, "lfence") +CK_PR_FENCE(load_store, "mfence") +CK_PR_FENCE(store, "sfence") +CK_PR_FENCE(store_load, "mfence") +CK_PR_FENCE(memory, "mfence") +CK_PR_FENCE(release, "mfence") +CK_PR_FENCE(acquire, "mfence") +CK_PR_FENCE(acqrel, "mfence") +CK_PR_FENCE(lock, "mfence") +CK_PR_FENCE(unlock, "mfence") + +#undef CK_PR_FENCE + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgl") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movl") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : CK_CC_IMM "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, char, "movl") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(char, char, "movb") +CK_PR_STORE_S(uint, unsigned int, "movl") +CK_PR_STORE_S(int, int, "movl") +CK_PR_STORE_S(32, uint32_t, "movl") +CK_PR_STORE_S(16, uint16_t, "movw") +CK_PR_STORE_S(8, uint8_t, "movb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddl") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "l") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : CK_CC_IMM "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I) CK_PR_BINARY(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "l") \ + CK_PR_BINARY_S(K, char, char, #K "b") \ + CK_PR_BINARY_S(K, int, int, #K "l") \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgl") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return (bool)z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "l", "eax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return (bool)c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint32_t, char, #K "l %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_H */ + diff --git a/include/gcc/x86_64/ck_f_pr.h b/include/gcc/x86_64/ck_f_pr.h new file mode 100644 index 0000000..545f5fd --- /dev/null +++ b/include/gcc/x86_64/ck_f_pr.h @@ -0,0 +1,202 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_64 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_64 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_64 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_8 +#define CK_F_PR_CAS_16_8_VALUE +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_4 +#define CK_F_PR_CAS_32_4_VALUE +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_16 +#define CK_F_PR_CAS_8_16_VALUE +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_16 +#define CK_F_PR_CAS_CHAR_16_VALUE +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_4 +#define CK_F_PR_CAS_INT_4_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_2 +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_4 +#define CK_F_PR_CAS_UINT_4_VALUE +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_16_8 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_32_4 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_64_2 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_8_16 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_CHAR_16 +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_INT_4 +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_PTR_2 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_UINT_4 +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + diff --git a/include/gcc/x86_64/ck_pr.h b/include/gcc/x86_64/ck_pr.h new file mode 100644 index 0000000..532d593 --- /dev/null +++ b/include/gcc/x86_64/ck_pr.h @@ -0,0 +1,585 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_64_H +#define CK_PR_X86_64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include <ck_cc.h> +#include <ck_md.h> +#include <ck_stdint.h> + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Support for TSX extensions. + */ +#ifdef CK_MD_RTM_ENABLE +#include "ck_pr_rtm.h" +#endif + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#else +#define CK_PR_LOCK_PREFIX "lock " +#endif + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) + * or "predefined delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "sfence") +CK_PR_FENCE(atomic_store, "sfence") +CK_PR_FENCE(atomic_load, "mfence") +CK_PR_FENCE(store_atomic, "sfence") +CK_PR_FENCE(load_atomic, "mfence") +CK_PR_FENCE(load, "lfence") +CK_PR_FENCE(load_store, "mfence") +CK_PR_FENCE(store, "sfence") +CK_PR_FENCE(store_load, "mfence") +CK_PR_FENCE(memory, "mfence") +CK_PR_FENCE(release, "mfence") +CK_PR_FENCE(acquire, "mfence") +CK_PR_FENCE(acqrel, "mfence") +CK_PR_FENCE(lock, "mfence") +CK_PR_FENCE(unlock, "mfence") + +#undef CK_PR_FENCE + +/* + * Read for ownership. Older compilers will generate the 32-bit + * 3DNow! variant which is binary compatible with x86-64 variant + * of prefetchw. + */ +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + __asm__ __volatile__("prefetchw (%0)" + : + : "r" (m) + : "memory"); + + return; +} +#endif /* CK_F_PR_RFO */ + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgq") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS_S(double, double, "xchgq") +#endif +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(64, uint64_t, "xchgq") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +/* + * Atomic load-from-memory operations. + */ +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movq") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "movq") +#endif +CK_PR_LOAD_S(64, uint64_t, "movq") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_load_64_2(const uint64_t target[2], uint64_t v[2]) +{ + __asm__ __volatile__("movq %%rdx, %%rcx;" + "movq %%rax, %%rbx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %2;" + : "=a" (v[0]), + "=d" (v[1]) + : "m" (*(const uint64_t *)target) + : "rbx", "rcx", "memory", "cc"); + return; +} + +CK_CC_INLINE static void +ck_pr_load_ptr_2(const void *t, void *v) +{ + ck_pr_load_64_2(CK_CPP_CAST(const uint64_t *, t), + CK_CPP_CAST(uint64_t *, v)); + return; +} + +#define CK_PR_LOAD_2(S, W, T) \ + CK_CC_INLINE static void \ + ck_pr_md_load_##S##_##W(const T t[2], T v[2]) \ + { \ + ck_pr_load_64_2((const uint64_t *)(const void *)t, \ + (uint64_t *)(void *)v); \ + return; \ + } + +CK_PR_LOAD_2(char, 16, char) +CK_PR_LOAD_2(int, 4, int) +CK_PR_LOAD_2(uint, 4, unsigned int) +CK_PR_LOAD_2(32, 4, uint32_t) +CK_PR_LOAD_2(16, 8, uint16_t) +CK_PR_LOAD_2(8, 16, uint8_t) + +#undef CK_PR_LOAD_2 + +/* + * Atomic store-to-memory operations. + */ +#define CK_PR_STORE_IMM(S, M, T, C, I, K) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : K "q" (v) \ + : "memory"); \ + return; \ + } + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_IMM(ptr, void, const void *, char, "movq", CK_CC_IMM_U32) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE(double, double, double, double, "movq") +#endif + +#define CK_PR_STORE_S(S, T, I, K) CK_PR_STORE_IMM(S, T, T, T, I, K) + +CK_PR_STORE_S(char, char, "movb", CK_CC_IMM_S32) +CK_PR_STORE_S(int, int, "movl", CK_CC_IMM_S32) +CK_PR_STORE_S(uint, unsigned int, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(64, uint64_t, "movq", CK_CC_IMM_U32) +CK_PR_STORE_S(32, uint32_t, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(16, uint16_t, "movw", CK_CC_IMM_U32) +CK_PR_STORE_S(8, uint8_t, "movb", CK_CC_IMM_U32) + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_IMM +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddq") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(64, uint64_t, "xaddq") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "q") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 64, uint64_t, #K "q") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I, O) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : O "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I, O) CK_PR_BINARY(K, S, T, T, T, I, O) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, char, char, #K "b", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, int, int, #K "l", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 64, uint64_t, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b", CK_CC_IMM_U32) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgq") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "cmpxchgq") +#endif +CK_PR_CAS_S(64, uint64_t, "cmpxchgq") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "q", "rax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_O_S(double, double, "q", "rax") +#endif +CK_PR_CAS_O_S(64, uint64_t, "q", "rax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Contrary to C-interface, alignment requirements are that of uint64_t[2]. + */ +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + bool z; + + __asm__ __volatile__("movq 0(%4), %%rax;" + "movq 8(%4), %%rdx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %0; setz %1" + : "+m" (*target), + "=q" (z) + : "b" (set[0]), + "c" (set[1]), + "q" (compare) + : "memory", "cc", "%rax", "%rdx"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *t, void *c, void *s) +{ + return ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, t), + CK_CPP_CAST(uint64_t *, c), + CK_CPP_CAST(uint64_t *, s)); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], + uint64_t compare[2], + uint64_t set[2], + uint64_t v[2]) +{ + bool z; + + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg16b %0;" + "setz %3" + : "+m" (*target), + "=a" (v[0]), + "=d" (v[1]), + "=q" (z) + : "a" (compare[0]), + "d" (compare[1]), + "b" (set[0]), + "c" (set[1]) + : "memory", "cc"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *t, void *c, void *s, void *v) +{ + return ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *,t), + CK_CPP_CAST(uint64_t *,c), + CK_CPP_CAST(uint64_t *,s), + CK_CPP_CAST(uint64_t *,v)); +} + +#define CK_PR_CAS_V(S, W, T) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W(T t[W], T c[W], T s[W]) \ +{ \ + return ck_pr_cas_64_2((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s); \ +} \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W##_value(T *t, T c[W], T s[W], T *v) \ +{ \ + return ck_pr_cas_64_2_value((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s, \ + (uint64_t *)(void *)v); \ +} + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_V(double, 2, double) +#endif +CK_PR_CAS_V(char, 16, char) +CK_PR_CAS_V(int, 4, int) +CK_PR_CAS_V(uint, 4, unsigned int) +CK_PR_CAS_V(32, 4, uint32_t) +CK_PR_CAS_V(16, 8, uint16_t) +CK_PR_CAS_V(8, 16, uint8_t) + +#undef CK_PR_CAS_V + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint64_t, char, #K "q %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 64, uint64_t, #K "q %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_64_H */ + diff --git a/include/gcc/x86_64/ck_pr_rtm.h b/include/gcc/x86_64/ck_pr_rtm.h new file mode 100644 index 0000000..45c7b9d --- /dev/null +++ b/include/gcc/x86_64/ck_pr_rtm.h @@ -0,0 +1,109 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Copyright (c) 2012,2013 Intel Corporation + * Author: Andi Kleen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +#ifndef CK_PR_X86_64_RTM_H +#define CK_PR_X86_64_RTM_H + +#ifndef CK_PR_X86_64_H +#error Do not include this file directly, use ck_pr.h +#endif + +#define CK_F_PR_RTM + +#include <ck_cc.h> +#include <ck_stdbool.h> + +#define CK_PR_RTM_STARTED (~0U) +#define CK_PR_RTM_EXPLICIT (1 << 0) +#define CK_PR_RTM_RETRY (1 << 1) +#define CK_PR_RTM_CONFLICT (1 << 2) +#define CK_PR_RTM_CAPACITY (1 << 3) +#define CK_PR_RTM_DEBUG (1 << 4) +#define CK_PR_RTM_NESTED (1 << 5) +#define CK_PR_RTM_CODE(x) (((x) >> 24) & 0xFF) + +CK_CC_INLINE static unsigned int +ck_pr_rtm_begin(void) +{ + unsigned int r = CK_PR_RTM_STARTED; + + __asm__ __volatile__(".byte 0xc7,0xf8;" + ".long 0;" + : "+a" (r) + : + : "memory"); + + return r; +} + +CK_CC_INLINE static void +ck_pr_rtm_end(void) +{ + + __asm__ __volatile__(".byte 0x0f,0x01,0xd5" ::: "memory"); + return; +} + +CK_CC_INLINE static void +ck_pr_rtm_abort(const unsigned int status) +{ + + __asm__ __volatile__(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory"); + return; +} + +CK_CC_INLINE static bool +ck_pr_rtm_test(void) +{ + bool r; + + __asm__ __volatile__(".byte 0x0f,0x01,0xd6;" + "setnz %0" + : "=r" (r) + : + : "memory"); + + return r; +} + +#endif /* CK_PR_X86_64_RTM_H */ + diff --git a/include/spinlock/anderson.h b/include/spinlock/anderson.h new file mode 100644 index 0000000..bebc5d8 --- /dev/null +++ b/include/spinlock/anderson.h @@ -0,0 +1,167 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_ANDERSON_H +#define CK_SPINLOCK_ANDERSON_H + +#include <ck_cc.h> +#include <ck_limits.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +#ifndef CK_F_SPINLOCK_ANDERSON +#define CK_F_SPINLOCK_ANDERSON +/* + * This is an implementation of Anderson's array-based queuing lock. + */ +struct ck_spinlock_anderson_thread { + unsigned int locked; + unsigned int position; +}; +typedef struct ck_spinlock_anderson_thread ck_spinlock_anderson_thread_t; + +struct ck_spinlock_anderson { + struct ck_spinlock_anderson_thread *slots; + unsigned int count; + unsigned int wrap; + unsigned int mask; + char pad[CK_MD_CACHELINE - sizeof(unsigned int) * 3 - sizeof(void *)]; + unsigned int next; +}; +typedef struct ck_spinlock_anderson ck_spinlock_anderson_t; + +CK_CC_INLINE static void +ck_spinlock_anderson_init(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread *slots, + unsigned int count) +{ + unsigned int i; + + slots[0].locked = false; + slots[0].position = 0; + for (i = 1; i < count; i++) { + slots[i].locked = true; + slots[i].position = i; + } + + lock->slots = slots; + lock->count = count; + lock->mask = count - 1; + lock->next = 0; + + /* + * If the number of threads is not a power of two then compute + * appropriate wrap-around value in the case of next slot counter + * overflow. + */ + if (count & (count - 1)) + lock->wrap = (UINT_MAX % count) + 1; + else + lock->wrap = 0; + + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_anderson_locked(struct ck_spinlock_anderson *lock) +{ + unsigned int position; + bool r; + + position = ck_pr_load_uint(&lock->next) & lock->mask; + r = ck_pr_load_uint(&lock->slots[position].locked); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_anderson_lock(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread **slot) +{ + unsigned int position, next; + unsigned int count = lock->count; + + /* + * If count is not a power of 2, then it is possible for an overflow + * to reallocate beginning slots to more than one thread. To avoid this + * use a compare-and-swap. + */ + if (lock->wrap != 0) { + position = ck_pr_load_uint(&lock->next); + + do { + if (position == UINT_MAX) + next = lock->wrap; + else + next = position + 1; + } while (ck_pr_cas_uint_value(&lock->next, position, + next, &position) == false); + + position %= count; + } else { + position = ck_pr_faa_uint(&lock->next, 1); + position &= lock->mask; + } + + /* Serialize with respect to previous thread's store. */ + ck_pr_fence_load(); + + /* + * Spin until slot is marked as unlocked. First slot is initialized to + * false. + */ + while (ck_pr_load_uint(&lock->slots[position].locked) == true) + ck_pr_stall(); + + /* Prepare slot for potential re-use by another thread. */ + ck_pr_store_uint(&lock->slots[position].locked, true); + ck_pr_fence_lock(); + + *slot = lock->slots + position; + return; +} + +CK_CC_INLINE static void +ck_spinlock_anderson_unlock(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread *slot) +{ + unsigned int position; + + ck_pr_fence_unlock(); + + /* Mark next slot as available. */ + if (lock->wrap == 0) + position = (slot->position + 1) & lock->mask; + else + position = (slot->position + 1) % lock->count; + + ck_pr_store_uint(&lock->slots[position].locked, false); + return; +} +#endif /* CK_F_SPINLOCK_ANDERSON */ +#endif /* CK_SPINLOCK_ANDERSON_H */ diff --git a/include/spinlock/cas.h b/include/spinlock/cas.h new file mode 100644 index 0000000..ff6d723 --- /dev/null +++ b/include/spinlock/cas.h @@ -0,0 +1,119 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_CAS_H +#define CK_SPINLOCK_CAS_H + +#include <ck_backoff.h> +#include <ck_cc.h> +#include <ck_elide.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +#ifndef CK_F_SPINLOCK_CAS +#define CK_F_SPINLOCK_CAS +/* + * This is a simple CACAS (TATAS) spinlock implementation. + */ +struct ck_spinlock_cas { + unsigned int value; +}; +typedef struct ck_spinlock_cas ck_spinlock_cas_t; + +#define CK_SPINLOCK_CAS_INITIALIZER {false} + +CK_CC_INLINE static void +ck_spinlock_cas_init(struct ck_spinlock_cas *lock) +{ + + lock->value = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_cas_trylock(struct ck_spinlock_cas *lock) +{ + unsigned int value; + + value = ck_pr_fas_uint(&lock->value, true); + ck_pr_fence_lock(); + return !value; +} + +CK_CC_INLINE static bool +ck_spinlock_cas_locked(struct ck_spinlock_cas *lock) +{ + bool r = ck_pr_load_uint(&lock->value); + + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_cas_lock(struct ck_spinlock_cas *lock) +{ + + while (ck_pr_cas_uint(&lock->value, false, true) == false) { + while (ck_pr_load_uint(&lock->value) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_cas_lock_eb(struct ck_spinlock_cas *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + + while (ck_pr_cas_uint(&lock->value, false, true) == false) + ck_backoff_eb(&backoff); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_cas_unlock(struct ck_spinlock_cas *lock) +{ + + /* Set lock state to unlocked. */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&lock->value, false); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_cas, ck_spinlock_cas_t, + ck_spinlock_cas_locked, ck_spinlock_cas_lock, + ck_spinlock_cas_locked, ck_spinlock_cas_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_cas, ck_spinlock_cas_t, + ck_spinlock_cas_locked, ck_spinlock_cas_trylock) + +#endif /* CK_F_SPINLOCK_CAS */ +#endif /* CK_SPINLOCK_CAS_H */ diff --git a/include/spinlock/clh.h b/include/spinlock/clh.h new file mode 100644 index 0000000..1133806 --- /dev/null +++ b/include/spinlock/clh.h @@ -0,0 +1,122 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_CLH_H +#define CK_SPINLOCK_CLH_H + +#include <ck_cc.h> +#include <ck_limits.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +#ifndef CK_F_SPINLOCK_CLH +#define CK_F_SPINLOCK_CLH + +struct ck_spinlock_clh { + unsigned int wait; + struct ck_spinlock_clh *previous; +}; +typedef struct ck_spinlock_clh ck_spinlock_clh_t; + +CK_CC_INLINE static void +ck_spinlock_clh_init(struct ck_spinlock_clh **lock, struct ck_spinlock_clh *unowned) +{ + + unowned->previous = NULL; + unowned->wait = false; + *lock = unowned; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_clh_locked(struct ck_spinlock_clh **queue) +{ + struct ck_spinlock_clh *head; + bool r; + + head = ck_pr_load_ptr(queue); + r = ck_pr_load_uint(&head->wait); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_clh_lock(struct ck_spinlock_clh **queue, struct ck_spinlock_clh *thread) +{ + struct ck_spinlock_clh *previous; + + /* Indicate to the next thread on queue that they will have to block. */ + thread->wait = true; + ck_pr_fence_store_atomic(); + + /* + * Mark current request as last request. Save reference to previous + * request. + */ + previous = ck_pr_fas_ptr(queue, thread); + thread->previous = previous; + + /* Wait until previous thread is done with lock. */ + ck_pr_fence_load(); + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_clh_unlock(struct ck_spinlock_clh **thread) +{ + struct ck_spinlock_clh *previous; + + /* + * If there are waiters, they are spinning on the current node wait + * flag. The flag is cleared so that the successor may complete an + * acquisition. If the caller is pre-empted then the predecessor field + * may be updated by a successor's lock operation. In order to avoid + * this, save a copy of the predecessor before setting the flag. + */ + previous = thread[0]->previous; + + /* + * We have to pay this cost anyways, use it as a compiler barrier too. + */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&(*thread)->wait, false); + + /* + * Predecessor is guaranteed not to be spinning on previous request, + * so update caller to use previous structure. This allows successor + * all the time in the world to successfully read updated wait flag. + */ + *thread = previous; + return; +} +#endif /* CK_F_SPINLOCK_CLH */ +#endif /* CK_SPINLOCK_CLH_H */ diff --git a/include/spinlock/dec.h b/include/spinlock/dec.h new file mode 100644 index 0000000..11d36dd --- /dev/null +++ b/include/spinlock/dec.h @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_DEC_H +#define CK_SPINLOCK_DEC_H + +#include <ck_backoff.h> +#include <ck_cc.h> +#include <ck_elide.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +#ifndef CK_F_SPINLOCK_DEC +#define CK_F_SPINLOCK_DEC +/* + * This is similar to the CACAS lock but makes use of an atomic decrement + * operation to check if the lock value was decremented to 0 from 1. The + * idea is that a decrement operation is cheaper than a compare-and-swap. + */ +struct ck_spinlock_dec { + unsigned int value; +}; +typedef struct ck_spinlock_dec ck_spinlock_dec_t; + +#define CK_SPINLOCK_DEC_INITIALIZER {1} + +CK_CC_INLINE static void +ck_spinlock_dec_init(struct ck_spinlock_dec *lock) +{ + + lock->value = 1; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_dec_trylock(struct ck_spinlock_dec *lock) +{ + unsigned int value; + + value = ck_pr_fas_uint(&lock->value, 0); + ck_pr_fence_lock(); + return value == 1; +} + +CK_CC_INLINE static bool +ck_spinlock_dec_locked(struct ck_spinlock_dec *lock) +{ + bool r; + + r = ck_pr_load_uint(&lock->value) != 1; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_dec_lock(struct ck_spinlock_dec *lock) +{ + bool r; + + for (;;) { + /* + * Only one thread is guaranteed to decrement lock to 0. + * Overflow must be protected against. No more than + * UINT_MAX lock requests can happen while the lock is held. + */ + ck_pr_dec_uint_zero(&lock->value, &r); + if (r == true) + break; + + /* Load value without generating write cycles. */ + while (ck_pr_load_uint(&lock->value) != 1) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_dec_lock_eb(struct ck_spinlock_dec *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + bool r; + + for (;;) { + ck_pr_dec_uint_zero(&lock->value, &r); + if (r == true) + break; + + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_dec_unlock(struct ck_spinlock_dec *lock) +{ + + ck_pr_fence_unlock(); + + /* + * Unconditionally set lock value to 1 so someone can decrement lock + * to 0. + */ + ck_pr_store_uint(&lock->value, 1); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_dec, ck_spinlock_dec_t, + ck_spinlock_dec_locked, ck_spinlock_dec_lock, + ck_spinlock_dec_locked, ck_spinlock_dec_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_dec, ck_spinlock_dec_t, + ck_spinlock_dec_locked, ck_spinlock_dec_trylock) + +#endif /* CK_F_SPINLOCK_DEC */ +#endif /* CK_SPINLOCK_DEC_H */ diff --git a/include/spinlock/fas.h b/include/spinlock/fas.h new file mode 100644 index 0000000..4e6c123 --- /dev/null +++ b/include/spinlock/fas.h @@ -0,0 +1,118 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_FAS_H +#define CK_SPINLOCK_FAS_H + +#include <ck_backoff.h> +#include <ck_cc.h> +#include <ck_elide.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +#ifndef CK_F_SPINLOCK_FAS +#define CK_F_SPINLOCK_FAS + +struct ck_spinlock_fas { + unsigned int value; +}; +typedef struct ck_spinlock_fas ck_spinlock_fas_t; + +#define CK_SPINLOCK_FAS_INITIALIZER {false} + +CK_CC_INLINE static void +ck_spinlock_fas_init(struct ck_spinlock_fas *lock) +{ + + lock->value = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_fas_trylock(struct ck_spinlock_fas *lock) +{ + bool value; + + value = ck_pr_fas_uint(&lock->value, true); + ck_pr_fence_lock(); + + return !value; +} + +CK_CC_INLINE static bool +ck_spinlock_fas_locked(struct ck_spinlock_fas *lock) +{ + bool r; + + r = ck_pr_load_uint(&lock->value); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_fas_lock(struct ck_spinlock_fas *lock) +{ + + while (ck_pr_fas_uint(&lock->value, true) == true) { + while (ck_pr_load_uint(&lock->value) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_fas_lock_eb(struct ck_spinlock_fas *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + + while (ck_pr_fas_uint(&lock->value, true) == true) + ck_backoff_eb(&backoff); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_fas_unlock(struct ck_spinlock_fas *lock) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&lock->value, false); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_fas, ck_spinlock_fas_t, + ck_spinlock_fas_locked, ck_spinlock_fas_lock, + ck_spinlock_fas_locked, ck_spinlock_fas_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_fas, ck_spinlock_fas_t, + ck_spinlock_fas_locked, ck_spinlock_fas_trylock) + +#endif /* CK_F_SPINLOCK_FAS */ +#endif /* CK_SPINLOCK_FAS_H */ diff --git a/include/spinlock/hclh.h b/include/spinlock/hclh.h new file mode 100644 index 0000000..296448b --- /dev/null +++ b/include/spinlock/hclh.h @@ -0,0 +1,145 @@ +/* + * Copyright 2013-2015 Olivier Houchard + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_HCLH_H +#define CK_SPINLOCK_HCLH_H + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +#ifndef CK_F_SPINLOCK_HCLH +#define CK_F_SPINLOCK_HCLH +struct ck_spinlock_hclh { + unsigned int wait; + unsigned int splice; + int cluster_id; + struct ck_spinlock_hclh *previous; +}; +typedef struct ck_spinlock_hclh ck_spinlock_hclh_t; + +CK_CC_INLINE static void +ck_spinlock_hclh_init(struct ck_spinlock_hclh **lock, + struct ck_spinlock_hclh *unowned, + int cluster_id) +{ + + unowned->previous = NULL; + unowned->wait = false; + unowned->splice = false; + unowned->cluster_id = cluster_id; + *lock = unowned; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_hclh_locked(struct ck_spinlock_hclh **queue) +{ + struct ck_spinlock_hclh *head; + bool r; + + head = ck_pr_load_ptr(queue); + r = ck_pr_load_uint(&head->wait); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_hclh_lock(struct ck_spinlock_hclh **glob_queue, + struct ck_spinlock_hclh **local_queue, + struct ck_spinlock_hclh *thread) +{ + struct ck_spinlock_hclh *previous, *local_tail; + + /* Indicate to the next thread on queue that they will have to block. */ + thread->wait = true; + thread->splice = false; + thread->cluster_id = (*local_queue)->cluster_id; + + /* Serialize with respect to update of local queue. */ + ck_pr_fence_store_atomic(); + + /* Mark current request as last request. Save reference to previous request. */ + previous = ck_pr_fas_ptr(local_queue, thread); + thread->previous = previous; + + /* Wait until previous thread from the local queue is done with lock. */ + ck_pr_fence_load(); + if (previous->previous != NULL && + previous->cluster_id == thread->cluster_id) { + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + /* We're head of the global queue, we're done */ + if (ck_pr_load_uint(&previous->splice) == false) + return; + } + + /* Now we need to splice the local queue into the global queue. */ + local_tail = ck_pr_load_ptr(local_queue); + previous = ck_pr_fas_ptr(glob_queue, local_tail); + + ck_pr_store_uint(&local_tail->splice, true); + + /* Wait until previous thread from the global queue is done with lock. */ + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_hclh_unlock(struct ck_spinlock_hclh **thread) +{ + struct ck_spinlock_hclh *previous; + + /* + * If there are waiters, they are spinning on the current node wait + * flag. The flag is cleared so that the successor may complete an + * acquisition. If the caller is pre-empted then the predecessor field + * may be updated by a successor's lock operation. In order to avoid + * this, save a copy of the predecessor before setting the flag. + */ + previous = thread[0]->previous; + + /* We have to pay this cost anyways, use it as a compiler barrier too. */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&(*thread)->wait, false); + + /* + * Predecessor is guaranteed not to be spinning on previous request, + * so update caller to use previous structure. This allows successor + * all the time in the world to successfully read updated wait flag. + */ + *thread = previous; + return; +} +#endif /* CK_F_SPINLOCK_HCLH */ +#endif /* CK_SPINLOCK_HCLH_H */ diff --git a/include/spinlock/mcs.h b/include/spinlock/mcs.h new file mode 100644 index 0000000..262c720 --- /dev/null +++ b/include/spinlock/mcs.h @@ -0,0 +1,155 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_MCS_H +#define CK_SPINLOCK_MCS_H + +#include <ck_cc.h> +#include <ck_pr.h> +#include <ck_stdbool.h> +#include <ck_stddef.h> + +#ifndef CK_F_SPINLOCK_MCS +#define CK_F_SPINLOCK_MCS + +struct ck_spinlock_mcs { + unsigned int locked; + struct ck_spinlock_mcs *next; +}; +typedef struct ck_spinlock_mcs * ck_spinlock_mcs_t; +typedef struct ck_spinlock_mcs ck_spinlock_mcs_context_t; + +#define CK_SPINLOCK_MCS_INITIALIZER (NULL) + +CK_CC_INLINE static void +ck_spinlock_mcs_init(struct ck_spinlock_mcs **queue) +{ + + *queue = NULL; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_mcs_trylock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + bool r; + + node->locked = true; + node->next = NULL; + ck_pr_fence_store_atomic(); + + r = ck_pr_cas_ptr(queue, NULL, node); + ck_pr_fence_lock(); + return r; +} + +CK_CC_INLINE static bool +ck_spinlock_mcs_locked(struct ck_spinlock_mcs **queue) +{ + bool r; + + r = ck_pr_load_ptr(queue) != NULL; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_mcs_lock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + struct ck_spinlock_mcs *previous; + + /* + * In the case that there is a successor, let them know they must + * wait for us to unlock. + */ + node->locked = true; + node->next = NULL; + ck_pr_fence_store_atomic(); + + /* + * Swap current tail with current lock request. If the swap operation + * returns NULL, it means the queue was empty. If the queue was empty, + * then the operation is complete. + */ + previous = ck_pr_fas_ptr(queue, node); + if (previous != NULL) { + /* + * Let the previous lock holder know that we are waiting on + * them. + */ + ck_pr_store_ptr(&previous->next, node); + while (ck_pr_load_uint(&node->locked) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_mcs_unlock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + struct ck_spinlock_mcs *next; + + ck_pr_fence_unlock(); + + next = ck_pr_load_ptr(&node->next); + if (next == NULL) { + /* + * If there is no request following us then it is a possibilty + * that we are the current tail. In this case, we may just + * mark the spinlock queue as empty. + */ + if (ck_pr_load_ptr(queue) == node && + ck_pr_cas_ptr(queue, node, NULL) == true) { + return; + } + + /* + * If the node is not the current tail then a lock operation + * is in-progress. In this case, busy-wait until the queue is + * in a consistent state to wake up the incoming lock + * request. + */ + for (;;) { + next = ck_pr_load_ptr(&node->next); + if (next != NULL) + break; + + ck_pr_stall(); + } + } + + /* Allow the next lock operation to complete. */ + ck_pr_store_uint(&next->locked, false); + return; +} +#endif /* CK_F_SPINLOCK_MCS */ +#endif /* CK_SPINLOCK_MCS_H */ diff --git a/include/spinlock/ticket.h b/include/spinlock/ticket.h new file mode 100644 index 0000000..3358547 --- /dev/null +++ b/include/spinlock/ticket.h @@ -0,0 +1,296 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_TICKET_H +#define CK_SPINLOCK_TICKET_H + +#include <ck_backoff.h> +#include <ck_cc.h> +#include <ck_elide.h> +#include <ck_md.h> +#include <ck_pr.h> +#include <ck_stdbool.h> + +#ifndef CK_F_SPINLOCK_TICKET +#define CK_F_SPINLOCK_TICKET +/* + * If 16-bit or 32-bit increment is supported, implement support for + * trylock functionality on availability of 32-bit or 64-bit fetch-and-add + * and compare-and-swap. This code path is only applied to x86*. + */ +#if defined(CK_MD_TSO) && (defined(__x86__) || defined(__x86_64__)) +#if defined(CK_F_PR_FAA_32) && defined(CK_F_PR_INC_16) && defined(CK_F_PR_CAS_32) +#define CK_SPINLOCK_TICKET_TYPE uint32_t +#define CK_SPINLOCK_TICKET_TYPE_BASE uint16_t +#define CK_SPINLOCK_TICKET_INC(x) ck_pr_inc_16(x) +#define CK_SPINLOCK_TICKET_CAS(x, y, z) ck_pr_cas_32(x, y, z) +#define CK_SPINLOCK_TICKET_FAA(x, y) ck_pr_faa_32(x, y) +#define CK_SPINLOCK_TICKET_LOAD(x) ck_pr_load_32(x) +#define CK_SPINLOCK_TICKET_INCREMENT (0x00010000UL) +#define CK_SPINLOCK_TICKET_MASK (0xFFFFUL) +#define CK_SPINLOCK_TICKET_SHIFT (16) +#elif defined(CK_F_PR_FAA_64) && defined(CK_F_PR_INC_32) && defined(CK_F_PR_CAS_64) +#define CK_SPINLOCK_TICKET_TYPE uint64_t +#define CK_SPINLOCK_TICKET_TYPE_BASE uint32_t +#define CK_SPINLOCK_TICKET_INC(x) ck_pr_inc_32(x) +#define CK_SPINLOCK_TICKET_CAS(x, y, z) ck_pr_cas_64(x, y, z) +#define CK_SPINLOCK_TICKET_FAA(x, y) ck_pr_faa_64(x, y) +#define CK_SPINLOCK_TICKET_LOAD(x) ck_pr_load_64(x) +#define CK_SPINLOCK_TICKET_INCREMENT (0x0000000100000000ULL) +#define CK_SPINLOCK_TICKET_MASK (0xFFFFFFFFULL) +#define CK_SPINLOCK_TICKET_SHIFT (32) +#endif +#endif /* CK_MD_TSO */ + +#if defined(CK_SPINLOCK_TICKET_TYPE) +#define CK_F_SPINLOCK_TICKET_TRYLOCK + +struct ck_spinlock_ticket { + CK_SPINLOCK_TICKET_TYPE value; +}; +typedef struct ck_spinlock_ticket ck_spinlock_ticket_t; +#define CK_SPINLOCK_TICKET_INITIALIZER { .value = 0 } + +CK_CC_INLINE static void +ck_spinlock_ticket_init(struct ck_spinlock_ticket *ticket) +{ + + ticket->value = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + + request = CK_SPINLOCK_TICKET_LOAD(&ticket->value); + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + ck_pr_fence_acquire(); + return request != position; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + + /* Get our ticket number and set next ticket number. */ + request = CK_SPINLOCK_TICKET_FAA(&ticket->value, + CK_SPINLOCK_TICKET_INCREMENT); + + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + while (request != position) { + ck_pr_stall(); + position = CK_SPINLOCK_TICKET_LOAD(&ticket->value) & + CK_SPINLOCK_TICKET_MASK; + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + ck_backoff_t backoff; + + /* Get our ticket number and set next ticket number. */ + request = CK_SPINLOCK_TICKET_FAA(&ticket->value, + CK_SPINLOCK_TICKET_INCREMENT); + + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + while (request != position) { + ck_pr_stall(); + position = CK_SPINLOCK_TICKET_LOAD(&ticket->value) & + CK_SPINLOCK_TICKET_MASK; + + backoff = (request - position) & CK_SPINLOCK_TICKET_MASK; + backoff <<= c; + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_trylock(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE snapshot, request, position; + + snapshot = CK_SPINLOCK_TICKET_LOAD(&ticket->value); + position = snapshot & CK_SPINLOCK_TICKET_MASK; + request = snapshot >> CK_SPINLOCK_TICKET_SHIFT; + + if (position != request) + return false; + + if (CK_SPINLOCK_TICKET_CAS(&ticket->value, + snapshot, snapshot + CK_SPINLOCK_TICKET_INCREMENT) == false) { + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket) +{ + + ck_pr_fence_unlock(); + CK_SPINLOCK_TICKET_INC((CK_SPINLOCK_TICKET_TYPE_BASE *)(void *)&ticket->value); + return; +} + +#undef CK_SPINLOCK_TICKET_TYPE +#undef CK_SPINLOCK_TICKET_TYPE_BASE +#undef CK_SPINLOCK_TICKET_INC +#undef CK_SPINLOCK_TICKET_FAA +#undef CK_SPINLOCK_TICKET_LOAD +#undef CK_SPINLOCK_TICKET_INCREMENT +#undef CK_SPINLOCK_TICKET_MASK +#undef CK_SPINLOCK_TICKET_SHIFT +#else +/* + * MESI benefits from cacheline padding between next and current. This avoids + * invalidation of current from the cache due to incoming lock requests. + */ +struct ck_spinlock_ticket { + unsigned int next; + unsigned int position; +}; +typedef struct ck_spinlock_ticket ck_spinlock_ticket_t; + +#define CK_SPINLOCK_TICKET_INITIALIZER {.next = 0, .position = 0} + +CK_CC_INLINE static void +ck_spinlock_ticket_init(struct ck_spinlock_ticket *ticket) +{ + + ticket->next = 0; + ticket->position = 0; + ck_pr_barrier(); + + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket) +{ + bool r; + + r = ck_pr_load_uint(&ticket->position) != + ck_pr_load_uint(&ticket->next); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket) +{ + unsigned int request; + + /* Get our ticket number and set next ticket number. */ + request = ck_pr_faa_uint(&ticket->next, 1); + + /* + * Busy-wait until our ticket number is current. + * We can get away without a fence here assuming + * our position counter does not overflow. + */ + while (ck_pr_load_uint(&ticket->position) != request) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c) +{ + ck_backoff_t backoff; + unsigned int request, position; + + request = ck_pr_faa_uint(&ticket->next, 1); + + for (;;) { + position = ck_pr_load_uint(&ticket->position); + if (position == request) + break; + + backoff = request - position; + backoff <<= c; + + /* + * Ideally, back-off from generating cache traffic for at least + * the amount of time necessary for the number of pending lock + * acquisition and relinquish operations (assuming an empty + * critical section). + */ + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket) +{ + unsigned int update; + + ck_pr_fence_unlock(); + + /* + * Update current ticket value so next lock request can proceed. + * Overflow behavior is assumed to be roll-over, in which case, + * it is only an issue if there are 2^32 pending lock requests. + */ + update = ck_pr_load_uint(&ticket->position); + ck_pr_store_uint(&ticket->position, update + 1); + return; +} +#endif /* !CK_F_SPINLOCK_TICKET_TRYLOCK */ + +CK_ELIDE_PROTOTYPE(ck_spinlock_ticket, ck_spinlock_ticket_t, + ck_spinlock_ticket_locked, ck_spinlock_ticket_lock, + ck_spinlock_ticket_locked, ck_spinlock_ticket_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_ticket, ck_spinlock_ticket_t, + ck_spinlock_ticket_locked, ck_spinlock_ticket_trylock) + +#endif /* CK_F_SPINLOCK_TICKET */ +#endif /* CK_SPINLOCK_TICKET_H */ |