diff options
Diffstat (limited to 'scripts/atomic')
25 files changed, 1030 insertions, 0 deletions
diff --git a/scripts/atomic/atomic-tbl.sh b/scripts/atomic/atomic-tbl.sh new file mode 100755 index 000000000..81d5c3203 --- /dev/null +++ b/scripts/atomic/atomic-tbl.sh @@ -0,0 +1,186 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# helpers for dealing with atomics.tbl + +#meta_in(meta, match) +meta_in() +{ + case "$1" in + [$2]) return 0;; + esac + + return 1 +} + +#meta_has_ret(meta) +meta_has_ret() +{ + meta_in "$1" "bBiIfFlR" +} + +#meta_has_acquire(meta) +meta_has_acquire() +{ + meta_in "$1" "BFIlR" +} + +#meta_has_release(meta) +meta_has_release() +{ + meta_in "$1" "BFIRs" +} + +#meta_has_relaxed(meta) +meta_has_relaxed() +{ + meta_in "$1" "BFIR" +} + +#find_fallback_template(pfx, name, sfx, order) +find_fallback_template() +{ + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local base="" + local file="" + + # We may have fallbacks for a specific case (e.g. read_acquire()), or + # an entire class, e.g. *inc*(). + # + # Start at the most specific, and fall back to the most general. Once + # we find a specific fallback, don't bother looking for more. + for base in "${pfx}${name}${sfx}${order}" "${name}"; do + file="${ATOMICDIR}/fallbacks/${base}" + + if [ -f "${file}" ]; then + printf "${file}" + break + fi + done +} + +#gen_ret_type(meta, int) +gen_ret_type() { + local meta="$1"; shift + local int="$1"; shift + + case "${meta}" in + [sv]) printf "void";; + [bB]) printf "bool";; + [aiIfFlR]) printf "${int}";; + esac +} + +#gen_ret_stmt(meta) +gen_ret_stmt() +{ + if meta_has_ret "${meta}"; then + printf "return "; + fi +} + +# gen_param_name(arg) +gen_param_name() +{ + # strip off the leading 'c' for 'cv' + local name="${1#c}" + printf "${name#*:}" +} + +# gen_param_type(arg, int, atomic) +gen_param_type() +{ + local type="${1%%:*}"; shift + local int="$1"; shift + local atomic="$1"; shift + + case "${type}" in + i) type="${int} ";; + p) type="${int} *";; + v) type="${atomic}_t *";; + cv) type="const ${atomic}_t *";; + esac + + printf "${type}" +} + +#gen_param(arg, int, atomic) +gen_param() +{ + local arg="$1"; shift + local int="$1"; shift + local atomic="$1"; shift + local name="$(gen_param_name "${arg}")" + local type="$(gen_param_type "${arg}" "${int}" "${atomic}")" + + printf "${type}${name}" +} + +#gen_params(int, atomic, arg...) +gen_params() +{ + local int="$1"; shift + local atomic="$1"; shift + + while [ "$#" -gt 0 ]; do + gen_param "$1" "${int}" "${atomic}" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_args(arg...) +gen_args() +{ + while [ "$#" -gt 0 ]; do + printf "$(gen_param_name "$1")" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_proto_order_variants(meta, pfx, name, sfx, ...) +gen_proto_order_variants() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + + if meta_has_acquire "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + fi + if meta_has_release "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + fi + if meta_has_relaxed "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" + fi +} + +#gen_proto_variants(meta, name, ...) +gen_proto_variants() +{ + local meta="$1"; shift + local name="$1"; shift + local pfx="" + local sfx="" + + meta_in "${meta}" "fF" && pfx="fetch_" + meta_in "${meta}" "R" && sfx="_return" + + gen_proto_order_variants "${meta}" "${pfx}" "${name}" "${sfx}" "$@" +} + +#gen_proto(meta, ...) +gen_proto() { + local meta="$1"; shift + for m in $(echo "${meta}" | grep -o .); do + gen_proto_variants "${m}" "$@" + done +} diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl new file mode 100644 index 000000000..fbee2f619 --- /dev/null +++ b/scripts/atomic/atomics.tbl @@ -0,0 +1,41 @@ +# name meta args... +# +# Where meta contains a string of variants to generate. +# Upper-case implies _{acquire,release,relaxed} variants. +# Valid meta values are: +# * B/b - bool: returns bool +# * v - void: returns void +# * I/i - int: returns base type +# * R - return: returns base type (has _return variants) +# * F/f - fetch: returns base type (has fetch_ variants) +# * l - load: returns base type (has _acquire order variant) +# * s - store: returns void (has _release order variant) +# +# Where args contains list of type[:name], where type is: +# * cv - const pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t) +# * v - pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t) +# * i - base type (int/s64/long) +# * p - pointer to base type (int/s64/long) +# +read l cv +set s v i +add vRF i v +sub vRF i v +inc vRF v +dec vRF v +and vF i v +andnot vF i v +or vF i v +xor vF i v +xchg I v i +cmpxchg I v i:old i:new +try_cmpxchg B v p:old i:new +sub_and_test b i v +dec_and_test b v +inc_and_test b v +add_negative b i v +add_unless fb v i:a i:u +inc_not_zero b v +inc_unless_negative b v +dec_unless_positive b v +dec_if_positive i v diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire new file mode 100755 index 000000000..ef764085c --- /dev/null +++ b/scripts/atomic/fallbacks/acquire @@ -0,0 +1,9 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}_acquire(${params}) +{ + ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + __atomic_acquire_fence(); + return ret; +} +EOF diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative new file mode 100755 index 000000000..15caa2eb2 --- /dev/null +++ b/scripts/atomic/fallbacks/add_negative @@ -0,0 +1,16 @@ +cat <<EOF +/** + * arch_${atomic}_add_negative - add and test if negative + * @i: integer value to add + * @v: pointer of type ${atomic}_t + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __always_inline bool +arch_${atomic}_add_negative(${int} i, ${atomic}_t *v) +{ + return arch_${atomic}_add_return(i, v) < 0; +} +EOF diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless new file mode 100755 index 000000000..9e5159c2c --- /dev/null +++ b/scripts/atomic/fallbacks/add_unless @@ -0,0 +1,16 @@ +cat << EOF +/** + * arch_${atomic}_add_unless - add unless the number is already a given value + * @v: pointer of type ${atomic}_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, if @v was not already @u. + * Returns true if the addition was done. + */ +static __always_inline bool +arch_${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) +{ + return arch_${atomic}_fetch_add_unless(v, a, u) != u; +} +EOF diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot new file mode 100755 index 000000000..5a42f54a3 --- /dev/null +++ b/scripts/atomic/fallbacks/andnot @@ -0,0 +1,7 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) +{ + ${retstmt}arch_${atomic}_${pfx}and${sfx}${order}(~i, v); +} +EOF diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec new file mode 100755 index 000000000..8c144c818 --- /dev/null +++ b/scripts/atomic/fallbacks/dec @@ -0,0 +1,7 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) +{ + ${retstmt}arch_${atomic}_${pfx}sub${sfx}${order}(1, v); +} +EOF diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test new file mode 100755 index 000000000..8549f359b --- /dev/null +++ b/scripts/atomic/fallbacks/dec_and_test @@ -0,0 +1,15 @@ +cat <<EOF +/** + * arch_${atomic}_dec_and_test - decrement and test + * @v: pointer of type ${atomic}_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __always_inline bool +arch_${atomic}_dec_and_test(${atomic}_t *v) +{ + return arch_${atomic}_dec_return(v) == 0; +} +EOF diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive new file mode 100755 index 000000000..86bdced34 --- /dev/null +++ b/scripts/atomic/fallbacks/dec_if_positive @@ -0,0 +1,15 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_dec_if_positive(${atomic}_t *v) +{ + ${int} dec, c = arch_${atomic}_read(v); + + do { + dec = c - 1; + if (unlikely(dec < 0)) + break; + } while (!arch_${atomic}_try_cmpxchg(v, &c, dec)); + + return dec; +} +EOF diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive new file mode 100755 index 000000000..c531d5afe --- /dev/null +++ b/scripts/atomic/fallbacks/dec_unless_positive @@ -0,0 +1,14 @@ +cat <<EOF +static __always_inline bool +arch_${atomic}_dec_unless_positive(${atomic}_t *v) +{ + ${int} c = arch_${atomic}_read(v); + + do { + if (unlikely(c > 0)) + return false; + } while (!arch_${atomic}_try_cmpxchg(v, &c, c - 1)); + + return true; +} +EOF diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence new file mode 100755 index 000000000..07757d8e3 --- /dev/null +++ b/scripts/atomic/fallbacks/fence @@ -0,0 +1,11 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}(${params}) +{ + ${ret} ret; + __atomic_pre_full_fence(); + ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + __atomic_post_full_fence(); + return ret; +} +EOF diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless new file mode 100755 index 000000000..68ce13c8b --- /dev/null +++ b/scripts/atomic/fallbacks/fetch_add_unless @@ -0,0 +1,23 @@ +cat << EOF +/** + * arch_${atomic}_fetch_add_unless - add unless the number is already a given value + * @v: pointer of type ${atomic}_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as @v was not already @u. + * Returns original value of @v + */ +static __always_inline ${int} +arch_${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) +{ + ${int} c = arch_${atomic}_read(v); + + do { + if (unlikely(c == u)) + break; + } while (!arch_${atomic}_try_cmpxchg(v, &c, c + a)); + + return c; +} +EOF diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc new file mode 100755 index 000000000..3c2c37391 --- /dev/null +++ b/scripts/atomic/fallbacks/inc @@ -0,0 +1,7 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) +{ + ${retstmt}arch_${atomic}_${pfx}add${sfx}${order}(1, v); +} +EOF diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test new file mode 100755 index 000000000..0cf23fe1e --- /dev/null +++ b/scripts/atomic/fallbacks/inc_and_test @@ -0,0 +1,15 @@ +cat <<EOF +/** + * arch_${atomic}_inc_and_test - increment and test + * @v: pointer of type ${atomic}_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_${atomic}_inc_and_test(${atomic}_t *v) +{ + return arch_${atomic}_inc_return(v) == 0; +} +EOF diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero new file mode 100755 index 000000000..ed8a1f562 --- /dev/null +++ b/scripts/atomic/fallbacks/inc_not_zero @@ -0,0 +1,14 @@ +cat <<EOF +/** + * arch_${atomic}_inc_not_zero - increment unless the number is zero + * @v: pointer of type ${atomic}_t + * + * Atomically increments @v by 1, if @v is non-zero. + * Returns true if the increment was done. + */ +static __always_inline bool +arch_${atomic}_inc_not_zero(${atomic}_t *v) +{ + return arch_${atomic}_add_unless(v, 1, 0); +} +EOF diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative new file mode 100755 index 000000000..95d8ce482 --- /dev/null +++ b/scripts/atomic/fallbacks/inc_unless_negative @@ -0,0 +1,14 @@ +cat <<EOF +static __always_inline bool +arch_${atomic}_inc_unless_negative(${atomic}_t *v) +{ + ${int} c = arch_${atomic}_read(v); + + do { + if (unlikely(c < 0)) + return false; + } while (!arch_${atomic}_try_cmpxchg(v, &c, c + 1)); + + return true; +} +EOF diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire new file mode 100755 index 000000000..a0ea1d26e --- /dev/null +++ b/scripts/atomic/fallbacks/read_acquire @@ -0,0 +1,16 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_read_acquire(const ${atomic}_t *v) +{ + ${int} ret; + + if (__native_word(${atomic}_t)) { + ret = smp_load_acquire(&(v)->counter); + } else { + ret = arch_${atomic}_read(v); + __atomic_acquire_fence(); + } + + return ret; +} +EOF diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release new file mode 100755 index 000000000..b46feb56d --- /dev/null +++ b/scripts/atomic/fallbacks/release @@ -0,0 +1,8 @@ +cat <<EOF +static __always_inline ${ret} +arch_${atomic}_${pfx}${name}${sfx}_release(${params}) +{ + __atomic_release_fence(); + ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); +} +EOF diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release new file mode 100755 index 000000000..05cdb7f42 --- /dev/null +++ b/scripts/atomic/fallbacks/set_release @@ -0,0 +1,12 @@ +cat <<EOF +static __always_inline void +arch_${atomic}_set_release(${atomic}_t *v, ${int} i) +{ + if (__native_word(${atomic}_t)) { + smp_store_release(&(v)->counter, i); + } else { + __atomic_release_fence(); + arch_${atomic}_set(v, i); + } +} +EOF diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test new file mode 100755 index 000000000..260f37341 --- /dev/null +++ b/scripts/atomic/fallbacks/sub_and_test @@ -0,0 +1,16 @@ +cat <<EOF +/** + * arch_${atomic}_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type ${atomic}_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_${atomic}_sub_and_test(${int} i, ${atomic}_t *v) +{ + return arch_${atomic}_sub_return(i, v) == 0; +} +EOF diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg new file mode 100755 index 000000000..890f850ed --- /dev/null +++ b/scripts/atomic/fallbacks/try_cmpxchg @@ -0,0 +1,11 @@ +cat <<EOF +static __always_inline bool +arch_${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) +{ + ${int} r, o = *old; + r = arch_${atomic}_cmpxchg${order}(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +EOF diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh new file mode 100755 index 000000000..3a07695e3 --- /dev/null +++ b/scripts/atomic/gen-atomic-fallback.sh @@ -0,0 +1,245 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...) +gen_template_fallback() +{ + local template="$1"; shift + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local retstmt="$(gen_ret_stmt "${meta}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local args="$(gen_args "$@")" + + if [ ! -z "${template}" ]; then + printf "#ifndef ${atomicname}\n" + . ${template} + printf "#define ${atomicname} ${atomicname}\n" + printf "#endif\n\n" + fi +} + +#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...) +gen_proto_fallback() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" + gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" +} + +#gen_basic_fallbacks(basename) +gen_basic_fallbacks() +{ + local basename="$1"; shift +cat << EOF +#define ${basename}_acquire ${basename} +#define ${basename}_release ${basename} +#define ${basename}_relaxed ${basename} +EOF +} + +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1" + + local basename="arch_${atomic}_${pfx}${name}${sfx}" + + printf "#define ${basename}${order} ${basename}${order}\n" +} + +#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...) +gen_proto_order_variants() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local atomic="$1" + + local basename="arch_${atomic}_${pfx}${name}${sfx}" + + local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" + + # If we don't have relaxed atomics, then we don't bother with ordering fallbacks + # read_acquire and set_release need to be templated, though + if ! meta_has_relaxed "${meta}"; then + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + + if meta_has_acquire "${meta}"; then + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + fi + + if meta_has_release "${meta}"; then + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + fi + + return + fi + + printf "#ifndef ${basename}_relaxed\n" + + if [ ! -z "${template}" ]; then + printf "#ifdef ${basename}\n" + fi + + gen_basic_fallbacks "${basename}" + + if [ ! -z "${template}" ]; then + printf "#endif /* ${basename} */\n\n" + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" + fi + + printf "#else /* ${basename}_relaxed */\n\n" + + gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + + printf "#endif /* ${basename}_relaxed */\n\n" +} + +gen_order_fallbacks() +{ + local xchg="$1"; shift + +cat <<EOF + +#ifndef ${xchg}_acquire +#define ${xchg}_acquire(...) \\ + __atomic_op_acquire(${xchg}, __VA_ARGS__) +#endif + +#ifndef ${xchg}_release +#define ${xchg}_release(...) \\ + __atomic_op_release(${xchg}, __VA_ARGS__) +#endif + +#ifndef ${xchg} +#define ${xchg}(...) \\ + __atomic_op_fence(${xchg}, __VA_ARGS__) +#endif + +EOF +} + +gen_xchg_fallbacks() +{ + local xchg="$1"; shift + printf "#ifndef ${xchg}_relaxed\n" + + gen_basic_fallbacks ${xchg} + + printf "#else /* ${xchg}_relaxed */\n" + + gen_order_fallbacks ${xchg} + + printf "#endif /* ${xchg}_relaxed */\n\n" +} + +gen_try_cmpxchg_fallback() +{ + local cmpxchg="$1"; shift; + local order="$1"; shift; + +cat <<EOF +#ifndef arch_try_${cmpxchg}${order} +#define arch_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\ +({ \\ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\ + ___r = arch_${cmpxchg}${order}((_ptr), ___o, (_new)); \\ + if (unlikely(___r != ___o)) \\ + *___op = ___r; \\ + likely(___r == ___o); \\ +}) +#endif /* arch_try_${cmpxchg}${order} */ + +EOF +} + +gen_try_cmpxchg_fallbacks() +{ + local cmpxchg="$1"; shift; + + printf "#ifndef arch_try_${cmpxchg}_relaxed\n" + printf "#ifdef arch_try_${cmpxchg}\n" + + gen_basic_fallbacks "arch_try_${cmpxchg}" + + printf "#endif /* arch_try_${cmpxchg} */\n\n" + + for order in "" "_acquire" "_release" "_relaxed"; do + gen_try_cmpxchg_fallback "${cmpxchg}" "${order}" + done + + printf "#else /* arch_try_${cmpxchg}_relaxed */\n" + + gen_order_fallbacks "arch_try_${cmpxchg}" + + printf "#endif /* arch_try_${cmpxchg}_relaxed */\n\n" +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_FALLBACK_H +#define _LINUX_ATOMIC_FALLBACK_H + +#include <linux/compiler.h> + +EOF + +for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do + gen_xchg_fallbacks "${xchg}" +done + +for cmpxchg in "cmpxchg" "cmpxchg64"; do + gen_try_cmpxchg_fallbacks "${cmpxchg}" +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +cat <<EOF +#ifdef CONFIG_GENERIC_ATOMIC64 +#include <asm-generic/atomic64.h> +#endif + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +cat <<EOF +#endif /* _LINUX_ATOMIC_FALLBACK_H */ +EOF diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh new file mode 100755 index 000000000..77c06526a --- /dev/null +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -0,0 +1,190 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_param_check(meta, arg) +gen_param_check() +{ + local meta="$1"; shift + local arg="$1"; shift + local type="${arg%%:*}" + local name="$(gen_param_name "${arg}")" + local rw="write" + + case "${type#c}" in + i) return;; + esac + + if [ ${type#c} != ${type} ]; then + # We don't write to constant parameters. + rw="read" + elif [ "${meta}" != "s" ]; then + # An atomic RMW: if this parameter is not a constant, and this atomic is + # not just a 's'tore, this parameter is both read from and written to. + rw="read_write" + fi + + printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n" +} + +#gen_params_checks(meta, arg...) +gen_params_checks() +{ + local meta="$1"; shift + local order="$1"; shift + + if [ "${order}" = "_release" ]; then + printf "\tkcsan_release();\n" + elif [ -z "${order}" ] && ! meta_in "$meta" "slv"; then + # RMW with return value is fully ordered + printf "\tkcsan_mb();\n" + fi + + while [ "$#" -gt 0 ]; do + gen_param_check "$meta" "$1" + shift; + done +} + +#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local checks="$(gen_params_checks "${meta}" "${order}" "$@")" + local args="$(gen_args "$@")" + local retstmt="$(gen_ret_stmt "${meta}")" + +cat <<EOF +static __always_inline ${ret} +${atomicname}(${params}) +{ +${checks} + ${retstmt}arch_${atomicname}(${args}); +} +EOF + + printf "\n" +} + +gen_xchg() +{ + local xchg="$1"; shift + local order="$1"; shift + local mult="$1"; shift + + kcsan_barrier="" + if [ "${xchg%_local}" = "${xchg}" ]; then + case "$order" in + _release) kcsan_barrier="kcsan_release()" ;; + "") kcsan_barrier="kcsan_mb()" ;; + esac + fi + + if [ "${xchg%${xchg#try_cmpxchg}}" = "try_cmpxchg" ] ; then + +cat <<EOF +#define ${xchg}${order}(ptr, oldp, ...) \\ +({ \\ + typeof(ptr) __ai_ptr = (ptr); \\ + typeof(oldp) __ai_oldp = (oldp); \\ +EOF +[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" +cat <<EOF + instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_write(__ai_oldp, ${mult}sizeof(*__ai_oldp)); \\ + arch_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ +}) +EOF + + else + +cat <<EOF +#define ${xchg}${order}(ptr, ...) \\ +({ \\ + typeof(ptr) __ai_ptr = (ptr); \\ +EOF +[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" +cat <<EOF + instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + arch_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ +}) +EOF + + fi +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +/* + * This file provides wrappers with KASAN instrumentation for atomic operations. + * To use this functionality an arch's atomic.h file needs to define all + * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include + * this file at the end. This file provides atomic_read() that forwards to + * arch_atomic_read() for actual atomic operation. + * Note: if an arch atomic operation is implemented by means of other atomic + * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use + * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid + * double instrumentation. + */ +#ifndef _LINUX_ATOMIC_INSTRUMENTED_H +#define _LINUX_ATOMIC_INSTRUMENTED_H + +#include <linux/build_bug.h> +#include <linux/compiler.h> +#include <linux/instrumented.h> + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic_long" "long" ${args} +done + + +for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do + for order in "" "_acquire" "_release" "_relaxed"; do + gen_xchg "${xchg}" "${order}" "" + printf "\n" + done +done + +for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do + gen_xchg "${xchg}" "" "" + printf "\n" +done + +gen_xchg "cmpxchg_double" "" "2 * " + +printf "\n\n" + +gen_xchg "cmpxchg_double_local" "" "2 * " + +cat <<EOF + +#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ +EOF diff --git a/scripts/atomic/gen-atomic-long.sh b/scripts/atomic/gen-atomic-long.sh new file mode 100755 index 000000000..eda89cea6 --- /dev/null +++ b/scripts/atomic/gen-atomic-long.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_cast(arg, int, atomic) +gen_cast() +{ + local arg="$1"; shift + local int="$1"; shift + local atomic="$1"; shift + + [ "${arg%%:*}" = "p" ] || return + + printf "($(gen_param_type "${arg}" "${int}" "${atomic}"))" +} + +#gen_args_cast(int, atomic, arg...) +gen_args_cast() +{ + local int="$1"; shift + local atomic="$1"; shift + + while [ "$#" -gt 0 ]; do + local cast="$(gen_cast "$1" "${int}" "${atomic}")" + local arg="$(gen_param_name "$1")" + printf "${cast}${arg}" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local name="$1$2$3$4"; shift; shift; shift; shift + local atomic="$1"; shift + local int="$1"; shift + + local ret="$(gen_ret_type "${meta}" "long")" + local params="$(gen_params "long" "atomic_long" "$@")" + local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")" + local retstmt="$(gen_ret_stmt "${meta}")" + +cat <<EOF +static __always_inline ${ret} +arch_atomic_long_${name}(${params}) +{ + ${retstmt}arch_${atomic}_${name}(${argscast}); +} + +EOF +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_LONG_H +#define _LINUX_ATOMIC_LONG_H + +#include <linux/compiler.h> +#include <asm/types.h> + +#ifdef CONFIG_64BIT +typedef atomic64_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) +#define atomic_long_cond_read_acquire atomic64_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed +#else +typedef atomic_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) +#define atomic_long_cond_read_acquire atomic_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed +#endif + +#ifdef CONFIG_64BIT + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +cat <<EOF +#else /* CONFIG_64BIT */ + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +cat <<EOF +#endif /* CONFIG_64BIT */ +#endif /* _LINUX_ATOMIC_LONG_H */ +EOF diff --git a/scripts/atomic/gen-atomics.sh b/scripts/atomic/gen-atomics.sh new file mode 100755 index 000000000..5b98a8307 --- /dev/null +++ b/scripts/atomic/gen-atomics.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Generate atomic headers + +ATOMICDIR=$(dirname $0) +ATOMICTBL=${ATOMICDIR}/atomics.tbl +LINUXDIR=${ATOMICDIR}/../.. + +cat <<EOF | +gen-atomic-instrumented.sh linux/atomic/atomic-instrumented.h +gen-atomic-long.sh linux/atomic/atomic-long.h +gen-atomic-fallback.sh linux/atomic/atomic-arch-fallback.h +EOF +while read script header args; do + /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header} + HASH="$(sha1sum ${LINUXDIR}/include/${header})" + HASH="${HASH%% *}" + printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header} +done |