diff options
Diffstat (limited to 'scripts/atomic')
49 files changed, 1366 insertions, 0 deletions
diff --git a/scripts/atomic/atomic-tbl.sh b/scripts/atomic/atomic-tbl.sh new file mode 100755 index 0000000000..608ff39ebd --- /dev/null +++ b/scripts/atomic/atomic-tbl.sh @@ -0,0 +1,290 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# helpers for dealing with atomics.tbl + +#meta_in(meta, match) +meta_in() +{ + case "$1" in + [$2]) return 0;; + esac + + return 1 +} + +#meta_has_ret(meta) +meta_has_ret() +{ + meta_in "$1" "bBiIfFlR" +} + +#meta_has_acquire(meta) +meta_has_acquire() +{ + meta_in "$1" "BFIlR" +} + +#meta_has_release(meta) +meta_has_release() +{ + meta_in "$1" "BFIRs" +} + +#meta_has_relaxed(meta) +meta_has_relaxed() +{ + meta_in "$1" "BFIR" +} + +#meta_is_implicitly_relaxed(meta) +meta_is_implicitly_relaxed() +{ + meta_in "$1" "vls" +} + +#find_template(tmpltype, pfx, name, sfx, order) +find_template() +{ + local tmpltype="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local base="" + local file="" + + # We may have fallbacks for a specific case (e.g. read_acquire()), or + # an entire class, e.g. *inc*(). + # + # Start at the most specific, and fall back to the most general. Once + # we find a specific fallback, don't bother looking for more. + for base in "${pfx}${name}${sfx}${order}" "${pfx}${name}${sfx}" "${name}"; do + file="${ATOMICDIR}/${tmpltype}/${base}" + + if [ -f "${file}" ]; then + printf "${file}" + break + fi + done +} + +#find_fallback_template(pfx, name, sfx, order) +find_fallback_template() +{ + find_template "fallbacks" "$@" +} + +#find_kerneldoc_template(pfx, name, sfx, order) +find_kerneldoc_template() +{ + find_template "kerneldoc" "$@" +} + +#gen_ret_type(meta, int) +gen_ret_type() { + local meta="$1"; shift + local int="$1"; shift + + case "${meta}" in + [sv]) printf "void";; + [bB]) printf "bool";; + [aiIfFlR]) printf "${int}";; + esac +} + +#gen_ret_stmt(meta) +gen_ret_stmt() +{ + if meta_has_ret "${meta}"; then + printf "return "; + fi +} + +# gen_param_name(arg) +gen_param_name() +{ + # strip off the leading 'c' for 'cv' + local name="${1#c}" + printf "${name#*:}" +} + +# gen_param_type(arg, int, atomic) +gen_param_type() +{ + local type="${1%%:*}"; shift + local int="$1"; shift + local atomic="$1"; shift + + case "${type}" in + i) type="${int} ";; + p) type="${int} *";; + v) type="${atomic}_t *";; + cv) type="const ${atomic}_t *";; + esac + + printf "${type}" +} + +#gen_param(arg, int, atomic) +gen_param() +{ + local arg="$1"; shift + local int="$1"; shift + local atomic="$1"; shift + local name="$(gen_param_name "${arg}")" + local type="$(gen_param_type "${arg}" "${int}" "${atomic}")" + + printf "${type}${name}" +} + +#gen_params(int, atomic, arg...) +gen_params() +{ + local int="$1"; shift + local atomic="$1"; shift + + while [ "$#" -gt 0 ]; do + gen_param "$1" "${int}" "${atomic}" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_args(arg...) +gen_args() +{ + while [ "$#" -gt 0 ]; do + printf "$(gen_param_name "$1")" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_desc_return(meta) +gen_desc_return() +{ + local meta="$1"; shift + + case "${meta}" in + [v]) + printf "Return: Nothing." + ;; + [Ff]) + printf "Return: The original value of @v." + ;; + [R]) + printf "Return: The updated value of @v." + ;; + [l]) + printf "Return: The value of @v." + ;; + esac +} + +#gen_template_kerneldoc(template, class, meta, pfx, name, sfx, order, atomic, int, args...) +gen_template_kerneldoc() +{ + local template="$1"; shift + local class="$1"; shift + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local retstmt="$(gen_ret_stmt "${meta}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local args="$(gen_args "$@")" + local desc_order="" + local desc_instrumentation="" + local desc_return="" + + if [ ! -z "${order}" ]; then + desc_order="${order##_}" + elif meta_is_implicitly_relaxed "${meta}"; then + desc_order="relaxed" + else + desc_order="full" + fi + + if [ -z "${class}" ]; then + desc_noinstr="Unsafe to use in noinstr code; use raw_${atomicname}() there." + else + desc_noinstr="Safe to use in noinstr code; prefer ${atomicname}() elsewhere." + fi + + desc_return="$(gen_desc_return "${meta}")" + + . ${template} +} + +#gen_kerneldoc(class, meta, pfx, name, sfx, order, atomic, int, args...) +gen_kerneldoc() +{ + local class="$1"; shift + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + + local tmpl="$(find_kerneldoc_template "${pfx}" "${name}" "${sfx}" "${order}")" + if [ -z "${tmpl}" ]; then + printf "/*\n" + printf " * No kerneldoc available for ${class}${atomicname}\n" + printf " */\n" + else + gen_template_kerneldoc "${tmpl}" "${class}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" + fi +} + +#gen_proto_order_variants(meta, pfx, name, sfx, ...) +gen_proto_order_variants() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + + if meta_has_acquire "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + fi + if meta_has_release "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + fi + if meta_has_relaxed "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" + fi +} + +#gen_proto_variants(meta, name, ...) +gen_proto_variants() +{ + local meta="$1"; shift + local name="$1"; shift + local pfx="" + local sfx="" + + meta_in "${meta}" "fF" && pfx="fetch_" + meta_in "${meta}" "R" && sfx="_return" + + gen_proto_order_variants "${meta}" "${pfx}" "${name}" "${sfx}" "$@" +} + +#gen_proto(meta, ...) +gen_proto() { + local meta="$1"; shift + for m in $(echo "${meta}" | grep -o .); do + gen_proto_variants "${m}" "$@" + done +} diff --git a/scripts/atomic/atomics.tbl b/scripts/atomic/atomics.tbl new file mode 100644 index 0000000000..903946cbf1 --- /dev/null +++ b/scripts/atomic/atomics.tbl @@ -0,0 +1,41 @@ +# name meta args... +# +# Where meta contains a string of variants to generate. +# Upper-case implies _{acquire,release,relaxed} variants. +# Valid meta values are: +# * B/b - bool: returns bool +# * v - void: returns void +# * I/i - int: returns base type +# * R - return: returns base type (has _return variants) +# * F/f - fetch: returns base type (has fetch_ variants) +# * l - load: returns base type (has _acquire order variant) +# * s - store: returns void (has _release order variant) +# +# Where args contains list of type[:name], where type is: +# * cv - const pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t) +# * v - pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t) +# * i - base type (int/s64/long) +# * p - pointer to base type (int/s64/long) +# +read l cv +set s v i +add vRF i v +sub vRF i v +inc vRF v +dec vRF v +and vF i v +andnot vF i v +or vF i v +xor vF i v +xchg I v i:new +cmpxchg I v i:old i:new +try_cmpxchg B v p:old i:new +sub_and_test b i v +dec_and_test b v +inc_and_test b v +add_negative B i v +add_unless fb v i:a i:u +inc_not_zero b v +inc_unless_negative b v +dec_unless_positive b v +dec_if_positive i v diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire new file mode 100755 index 0000000000..4da0cab360 --- /dev/null +++ b/scripts/atomic/fallbacks/acquire @@ -0,0 +1,5 @@ +cat <<EOF + ${ret} ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + __atomic_acquire_fence(); + return ret; +EOF diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative new file mode 100755 index 0000000000..1d3d4ab3a9 --- /dev/null +++ b/scripts/atomic/fallbacks/add_negative @@ -0,0 +1,3 @@ +cat <<EOF + return raw_${atomic}_add_return${order}(i, v) < 0; +EOF diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless new file mode 100755 index 0000000000..95ecb2b740 --- /dev/null +++ b/scripts/atomic/fallbacks/add_unless @@ -0,0 +1,3 @@ +cat << EOF + return raw_${atomic}_fetch_add_unless(v, a, u) != u; +EOF diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot new file mode 100755 index 0000000000..66760457e6 --- /dev/null +++ b/scripts/atomic/fallbacks/andnot @@ -0,0 +1,3 @@ +cat <<EOF + ${retstmt}raw_${atomic}_${pfx}and${sfx}${order}(~i, v); +EOF diff --git a/scripts/atomic/fallbacks/cmpxchg b/scripts/atomic/fallbacks/cmpxchg new file mode 100644 index 0000000000..1c8507f62e --- /dev/null +++ b/scripts/atomic/fallbacks/cmpxchg @@ -0,0 +1,3 @@ +cat <<EOF + return raw_cmpxchg${order}(&v->counter, old, new); +EOF diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec new file mode 100755 index 0000000000..60d286d403 --- /dev/null +++ b/scripts/atomic/fallbacks/dec @@ -0,0 +1,3 @@ +cat <<EOF + ${retstmt}raw_${atomic}_${pfx}sub${sfx}${order}(1, v); +EOF diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test new file mode 100755 index 0000000000..3a0278e0dd --- /dev/null +++ b/scripts/atomic/fallbacks/dec_and_test @@ -0,0 +1,3 @@ +cat <<EOF + return raw_${atomic}_dec_return(v) == 0; +EOF diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive new file mode 100755 index 0000000000..f65c11b4b8 --- /dev/null +++ b/scripts/atomic/fallbacks/dec_if_positive @@ -0,0 +1,11 @@ +cat <<EOF + ${int} dec, c = raw_${atomic}_read(v); + + do { + dec = c - 1; + if (unlikely(dec < 0)) + break; + } while (!raw_${atomic}_try_cmpxchg(v, &c, dec)); + + return dec; +EOF diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive new file mode 100755 index 0000000000..d025361d7b --- /dev/null +++ b/scripts/atomic/fallbacks/dec_unless_positive @@ -0,0 +1,10 @@ +cat <<EOF + ${int} c = raw_${atomic}_read(v); + + do { + if (unlikely(c > 0)) + return false; + } while (!raw_${atomic}_try_cmpxchg(v, &c, c - 1)); + + return true; +EOF diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence new file mode 100755 index 0000000000..40d5b39765 --- /dev/null +++ b/scripts/atomic/fallbacks/fence @@ -0,0 +1,7 @@ +cat <<EOF + ${ret} ret; + __atomic_pre_full_fence(); + ret = arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + __atomic_post_full_fence(); + return ret; +EOF diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless new file mode 100755 index 0000000000..8db7e9e17f --- /dev/null +++ b/scripts/atomic/fallbacks/fetch_add_unless @@ -0,0 +1,10 @@ +cat << EOF + ${int} c = raw_${atomic}_read(v); + + do { + if (unlikely(c == u)) + break; + } while (!raw_${atomic}_try_cmpxchg(v, &c, c + a)); + + return c; +EOF diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc new file mode 100755 index 0000000000..56c770f591 --- /dev/null +++ b/scripts/atomic/fallbacks/inc @@ -0,0 +1,3 @@ +cat <<EOF + ${retstmt}raw_${atomic}_${pfx}add${sfx}${order}(1, v); +EOF diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test new file mode 100755 index 0000000000..7d16a10f22 --- /dev/null +++ b/scripts/atomic/fallbacks/inc_and_test @@ -0,0 +1,3 @@ +cat <<EOF + return raw_${atomic}_inc_return(v) == 0; +EOF diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero new file mode 100755 index 0000000000..1fcef1e55b --- /dev/null +++ b/scripts/atomic/fallbacks/inc_not_zero @@ -0,0 +1,3 @@ +cat <<EOF + return raw_${atomic}_add_unless(v, 1, 0); +EOF diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative new file mode 100755 index 0000000000..7b4b098688 --- /dev/null +++ b/scripts/atomic/fallbacks/inc_unless_negative @@ -0,0 +1,10 @@ +cat <<EOF + ${int} c = raw_${atomic}_read(v); + + do { + if (unlikely(c < 0)) + return false; + } while (!raw_${atomic}_try_cmpxchg(v, &c, c + 1)); + + return true; +EOF diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire new file mode 100755 index 0000000000..e319862d2f --- /dev/null +++ b/scripts/atomic/fallbacks/read_acquire @@ -0,0 +1,12 @@ +cat <<EOF + ${int} ret; + + if (__native_word(${atomic}_t)) { + ret = smp_load_acquire(&(v)->counter); + } else { + ret = raw_${atomic}_read(v); + __atomic_acquire_fence(); + } + + return ret; +EOF diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release new file mode 100755 index 0000000000..1e6daf57b4 --- /dev/null +++ b/scripts/atomic/fallbacks/release @@ -0,0 +1,4 @@ +cat <<EOF + __atomic_release_fence(); + ${retstmt}arch_${atomic}_${pfx}${name}${sfx}_relaxed(${args}); +EOF diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release new file mode 100755 index 0000000000..16a374ae6b --- /dev/null +++ b/scripts/atomic/fallbacks/set_release @@ -0,0 +1,8 @@ +cat <<EOF + if (__native_word(${atomic}_t)) { + smp_store_release(&(v)->counter, i); + } else { + __atomic_release_fence(); + raw_${atomic}_set(v, i); + } +EOF diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test new file mode 100755 index 0000000000..d1f746fe0c --- /dev/null +++ b/scripts/atomic/fallbacks/sub_and_test @@ -0,0 +1,3 @@ +cat <<EOF + return raw_${atomic}_sub_return(i, v) == 0; +EOF diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg new file mode 100755 index 0000000000..d4da82092b --- /dev/null +++ b/scripts/atomic/fallbacks/try_cmpxchg @@ -0,0 +1,7 @@ +cat <<EOF + ${int} r, o = *old; + r = raw_${atomic}_cmpxchg${order}(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +EOF diff --git a/scripts/atomic/fallbacks/xchg b/scripts/atomic/fallbacks/xchg new file mode 100644 index 0000000000..e4def1e0d0 --- /dev/null +++ b/scripts/atomic/fallbacks/xchg @@ -0,0 +1,3 @@ +cat <<EOF + return raw_xchg${order}(&v->counter, new); +EOF diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh new file mode 100755 index 0000000000..a45154cefa --- /dev/null +++ b/scripts/atomic/gen-atomic-fallback.sh @@ -0,0 +1,333 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...) +gen_template_fallback() +{ + local template="$1"; shift + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local ret="$(gen_ret_type "${meta}" "${int}")" + local retstmt="$(gen_ret_stmt "${meta}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local args="$(gen_args "$@")" + + . ${template} +} + +#gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...) +gen_order_fallback() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local tmpl_order=${order#_} + local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}" + gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" +} + +#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...) +gen_proto_fallback() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" + gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@" +} + +#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + local basename="${atomic}_${pfx}${name}${sfx}" + + local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local retstmt="$(gen_ret_stmt "${meta}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local args="$(gen_args "$@")" + + gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@" + + printf "static __always_inline ${ret}\n" + printf "raw_${atomicname}(${params})\n" + printf "{\n" + + # Where there is no possible fallback, this order variant is mandatory + # and must be provided by arch code. Add a comment to the header to + # make this obvious. + # + # Ideally we'd error on a missing definition, but arch code might + # define this order variant as a C function without a preprocessor + # symbol. + if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then + printf "\t${retstmt}arch_${atomicname}(${args});\n" + printf "}\n\n" + return + fi + + printf "#if defined(arch_${atomicname})\n" + printf "\t${retstmt}arch_${atomicname}(${args});\n" + + # Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops + if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then + printf "#elif defined(arch_${basename}_relaxed)\n" + gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@" + fi + + # Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops + if [ ! -z "${order}" ] && ! meta_is_implicitly_relaxed "${meta}"; then + printf "#elif defined(arch_${basename})\n" + printf "\t${retstmt}arch_${basename}(${args});\n" + fi + + printf "#else\n" + if [ ! -z "${template}" ]; then + gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@" + else + printf "#error \"Unable to define raw_${atomicname}\"\n" + fi + + printf "#endif\n" + printf "}\n\n" +} + + +#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...) +gen_proto_order_variants() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local atomic="$1" + + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" + + if meta_has_acquire "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" + fi + + if meta_has_release "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" + fi + + if meta_has_relaxed "${meta}"; then + gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@" + fi +} + +#gen_basic_fallbacks(basename) +gen_basic_fallbacks() +{ + local basename="$1"; shift +cat << EOF +#define raw_${basename}_acquire arch_${basename} +#define raw_${basename}_release arch_${basename} +#define raw_${basename}_relaxed arch_${basename} +EOF +} + +gen_order_fallbacks() +{ + local xchg="$1"; shift + +cat <<EOF + +#define raw_${xchg}_relaxed arch_${xchg}_relaxed + +#ifdef arch_${xchg}_acquire +#define raw_${xchg}_acquire arch_${xchg}_acquire +#else +#define raw_${xchg}_acquire(...) \\ + __atomic_op_acquire(arch_${xchg}, __VA_ARGS__) +#endif + +#ifdef arch_${xchg}_release +#define raw_${xchg}_release arch_${xchg}_release +#else +#define raw_${xchg}_release(...) \\ + __atomic_op_release(arch_${xchg}, __VA_ARGS__) +#endif + +#ifdef arch_${xchg} +#define raw_${xchg} arch_${xchg} +#else +#define raw_${xchg}(...) \\ + __atomic_op_fence(arch_${xchg}, __VA_ARGS__) +#endif + +EOF +} + +gen_xchg_order_fallback() +{ + local xchg="$1"; shift + local order="$1"; shift + local forder="${order:-_fence}" + + printf "#if defined(arch_${xchg}${order})\n" + printf "#define raw_${xchg}${order} arch_${xchg}${order}\n" + + if [ "${order}" != "_relaxed" ]; then + printf "#elif defined(arch_${xchg}_relaxed)\n" + printf "#define raw_${xchg}${order}(...) \\\\\n" + printf " __atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n" + fi + + if [ ! -z "${order}" ]; then + printf "#elif defined(arch_${xchg})\n" + printf "#define raw_${xchg}${order} arch_${xchg}\n" + fi + + printf "#else\n" + printf "extern void raw_${xchg}${order}_not_implemented(void);\n" + printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n" + printf "#endif\n\n" +} + +gen_xchg_fallbacks() +{ + local xchg="$1"; shift + + for order in "" "_acquire" "_release" "_relaxed"; do + gen_xchg_order_fallback "${xchg}" "${order}" + done +} + +gen_try_cmpxchg_fallback() +{ + local cmpxchg="$1"; shift; + local order="$1"; shift; + +cat <<EOF +#define raw_try_${cmpxchg}${order}(_ptr, _oldp, _new) \\ +({ \\ + typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\ + ___r = raw_${cmpxchg}${order}((_ptr), ___o, (_new)); \\ + if (unlikely(___r != ___o)) \\ + *___op = ___r; \\ + likely(___r == ___o); \\ +}) +EOF +} + +gen_try_cmpxchg_order_fallback() +{ + local cmpxchg="$1"; shift + local order="$1"; shift + local forder="${order:-_fence}" + + printf "#if defined(arch_try_${cmpxchg}${order})\n" + printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n" + + if [ "${order}" != "_relaxed" ]; then + printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n" + printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n" + printf " __atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n" + fi + + if [ ! -z "${order}" ]; then + printf "#elif defined(arch_try_${cmpxchg})\n" + printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n" + fi + + printf "#else\n" + gen_try_cmpxchg_fallback "${cmpxchg}" "${order}" + printf "#endif\n\n" +} + +gen_try_cmpxchg_fallbacks() +{ + local cmpxchg="$1"; shift; + + for order in "" "_acquire" "_release" "_relaxed"; do + gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}" + done +} + +gen_cmpxchg_local_fallbacks() +{ + local cmpxchg="$1"; shift + + printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n" + printf "#ifdef arch_try_${cmpxchg}\n" + printf "#define raw_try_${cmpxchg} arch_try_${cmpxchg}\n" + printf "#else\n" + gen_try_cmpxchg_fallback "${cmpxchg}" "" + printf "#endif\n\n" +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_FALLBACK_H +#define _LINUX_ATOMIC_FALLBACK_H + +#include <linux/compiler.h> + +EOF + +for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do + gen_xchg_fallbacks "${xchg}" +done + +for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do + gen_try_cmpxchg_fallbacks "${cmpxchg}" +done + +for cmpxchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local"; do + gen_cmpxchg_local_fallbacks "${cmpxchg}" "" +done + +for cmpxchg in "sync_cmpxchg"; do + printf "#define raw_${cmpxchg} arch_${cmpxchg}\n\n" +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +cat <<EOF +#ifdef CONFIG_GENERIC_ATOMIC64 +#include <asm-generic/atomic64.h> +#endif + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +cat <<EOF +#endif /* _LINUX_ATOMIC_FALLBACK_H */ +EOF diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh new file mode 100755 index 0000000000..8f8f8e3b20 --- /dev/null +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_param_check(meta, arg) +gen_param_check() +{ + local meta="$1"; shift + local arg="$1"; shift + local type="${arg%%:*}" + local name="$(gen_param_name "${arg}")" + local rw="write" + + case "${type#c}" in + i) return;; + esac + + if [ ${type#c} != ${type} ]; then + # We don't write to constant parameters. + rw="read" + elif [ "${meta}" != "s" ]; then + # An atomic RMW: if this parameter is not a constant, and this atomic is + # not just a 's'tore, this parameter is both read from and written to. + rw="read_write" + fi + + printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n" +} + +#gen_params_checks(meta, arg...) +gen_params_checks() +{ + local meta="$1"; shift + local order="$1"; shift + + if [ "${order}" = "_release" ]; then + printf "\tkcsan_release();\n" + elif [ -z "${order}" ] && ! meta_in "$meta" "slv"; then + # RMW with return value is fully ordered + printf "\tkcsan_mb();\n" + fi + + while [ "$#" -gt 0 ]; do + gen_param_check "$meta" "$1" + shift; + done +} + +#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + local atomic="$1"; shift + local int="$1"; shift + + local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "${int}")" + local params="$(gen_params "${int}" "${atomic}" "$@")" + local checks="$(gen_params_checks "${meta}" "${order}" "$@")" + local args="$(gen_args "$@")" + local retstmt="$(gen_ret_stmt "${meta}")" + + gen_kerneldoc "" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@" + +cat <<EOF +static __always_inline ${ret} +${atomicname}(${params}) +{ +${checks} + ${retstmt}raw_${atomicname}(${args}); +} +EOF + + printf "\n" +} + +gen_xchg() +{ + local xchg="$1"; shift + local order="$1"; shift + + kcsan_barrier="" + if [ "${xchg%_local}" = "${xchg}" ]; then + case "$order" in + _release) kcsan_barrier="kcsan_release()" ;; + "") kcsan_barrier="kcsan_mb()" ;; + esac + fi + + if [ "${xchg%${xchg#try_cmpxchg}}" = "try_cmpxchg" ] ; then + +cat <<EOF +#define ${xchg}${order}(ptr, oldp, ...) \\ +({ \\ + typeof(ptr) __ai_ptr = (ptr); \\ + typeof(oldp) __ai_oldp = (oldp); \\ +EOF +[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" +cat <<EOF + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\ + instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \\ + raw_${xchg}${order}(__ai_ptr, __ai_oldp, __VA_ARGS__); \\ +}) +EOF + + else + +cat <<EOF +#define ${xchg}${order}(ptr, ...) \\ +({ \\ + typeof(ptr) __ai_ptr = (ptr); \\ +EOF +[ -n "$kcsan_barrier" ] && printf "\t${kcsan_barrier}; \\\\\n" +cat <<EOF + instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \\ + raw_${xchg}${order}(__ai_ptr, __VA_ARGS__); \\ +}) +EOF + + fi +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +/* + * This file provoides atomic operations with explicit instrumentation (e.g. + * KASAN, KCSAN), which should be used unless it is necessary to avoid + * instrumentation. Where it is necessary to aovid instrumenation, the + * raw_atomic*() operations should be used. + */ +#ifndef _LINUX_ATOMIC_INSTRUMENTED_H +#define _LINUX_ATOMIC_INSTRUMENTED_H + +#include <linux/build_bug.h> +#include <linux/compiler.h> +#include <linux/instrumented.h> + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic" "int" ${args} +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} +done + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" "atomic_long" "long" ${args} +done + + +for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do + for order in "" "_acquire" "_release" "_relaxed"; do + gen_xchg "${xchg}" "${order}" + printf "\n" + done +done + +for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do + gen_xchg "${xchg}" "" + printf "\n" +done + +cat <<EOF + +#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ +EOF diff --git a/scripts/atomic/gen-atomic-long.sh b/scripts/atomic/gen-atomic-long.sh new file mode 100755 index 0000000000..9826be3ba9 --- /dev/null +++ b/scripts/atomic/gen-atomic-long.sh @@ -0,0 +1,100 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +ATOMICDIR=$(dirname $0) + +. ${ATOMICDIR}/atomic-tbl.sh + +#gen_cast(arg, int, atomic) +gen_cast() +{ + local arg="$1"; shift + local int="$1"; shift + local atomic="$1"; shift + + [ "${arg%%:*}" = "p" ] || return + + printf "($(gen_param_type "${arg}" "${int}" "${atomic}"))" +} + +#gen_args_cast(int, atomic, arg...) +gen_args_cast() +{ + local int="$1"; shift + local atomic="$1"; shift + + while [ "$#" -gt 0 ]; do + local cast="$(gen_cast "$1" "${int}" "${atomic}")" + local arg="$(gen_param_name "$1")" + printf "${cast}${arg}" + [ "$#" -gt 1 ] && printf ", " + shift; + done +} + +#gen_proto_order_variant(meta, pfx, name, sfx, order, arg...) +gen_proto_order_variant() +{ + local meta="$1"; shift + local pfx="$1"; shift + local name="$1"; shift + local sfx="$1"; shift + local order="$1"; shift + + local atomicname="${pfx}${name}${sfx}${order}" + + local ret="$(gen_ret_type "${meta}" "long")" + local params="$(gen_params "long" "atomic_long" "$@")" + local argscast_32="$(gen_args_cast "int" "atomic" "$@")" + local argscast_64="$(gen_args_cast "s64" "atomic64" "$@")" + local retstmt="$(gen_ret_stmt "${meta}")" + + gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "atomic_long" "long" "$@" + +cat <<EOF +static __always_inline ${ret} +raw_atomic_long_${atomicname}(${params}) +{ +#ifdef CONFIG_64BIT + ${retstmt}raw_atomic64_${atomicname}(${argscast_64}); +#else + ${retstmt}raw_atomic_${atomicname}(${argscast_32}); +#endif +} + +EOF +} + +cat << EOF +// SPDX-License-Identifier: GPL-2.0 + +// Generated by $0 +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_LONG_H +#define _LINUX_ATOMIC_LONG_H + +#include <linux/compiler.h> +#include <asm/types.h> + +#ifdef CONFIG_64BIT +typedef atomic64_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) +#define atomic_long_cond_read_acquire atomic64_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed +#else +typedef atomic_t atomic_long_t; +#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) +#define atomic_long_cond_read_acquire atomic_cond_read_acquire +#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed +#endif + +EOF + +grep '^[a-z]' "$1" | while read name meta args; do + gen_proto "${meta}" "${name}" ${args} +done + +cat <<EOF +#endif /* _LINUX_ATOMIC_LONG_H */ +EOF diff --git a/scripts/atomic/gen-atomics.sh b/scripts/atomic/gen-atomics.sh new file mode 100755 index 0000000000..5b98a83076 --- /dev/null +++ b/scripts/atomic/gen-atomics.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Generate atomic headers + +ATOMICDIR=$(dirname $0) +ATOMICTBL=${ATOMICDIR}/atomics.tbl +LINUXDIR=${ATOMICDIR}/../.. + +cat <<EOF | +gen-atomic-instrumented.sh linux/atomic/atomic-instrumented.h +gen-atomic-long.sh linux/atomic/atomic-long.h +gen-atomic-fallback.sh linux/atomic/atomic-arch-fallback.h +EOF +while read script header args; do + /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header} + HASH="$(sha1sum ${LINUXDIR}/include/${header})" + HASH="${HASH%% *}" + printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header} +done diff --git a/scripts/atomic/kerneldoc/add b/scripts/atomic/kerneldoc/add new file mode 100644 index 0000000000..991f3dafce --- /dev/null +++ b/scripts/atomic/kerneldoc/add @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic add with ${desc_order} ordering + * @i: ${int} value to add + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v + @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/add_negative b/scripts/atomic/kerneldoc/add_negative new file mode 100644 index 0000000000..f4ca1f05d1 --- /dev/null +++ b/scripts/atomic/kerneldoc/add_negative @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic add and test if negative with ${desc_order} ordering + * @i: ${int} value to add + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v + @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if the resulting value of @v is negative, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/add_unless b/scripts/atomic/kerneldoc/add_unless new file mode 100644 index 0000000000..f828e5f675 --- /dev/null +++ b/scripts/atomic/kerneldoc/add_unless @@ -0,0 +1,18 @@ +if [ -z "${pfx}" ]; then + desc_return="Return: @true if @v was updated, @false otherwise." +fi + +cat <<EOF +/** + * ${class}${atomicname}() - atomic add unless value with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * @a: ${int} value to add + * @u: ${int} value to compare with + * + * If (@v != @u), atomically updates @v to (@v + @a) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/and b/scripts/atomic/kerneldoc/and new file mode 100644 index 0000000000..a923574351 --- /dev/null +++ b/scripts/atomic/kerneldoc/and @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic bitwise AND with ${desc_order} ordering + * @i: ${int} value + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v & @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/andnot b/scripts/atomic/kerneldoc/andnot new file mode 100644 index 0000000000..64bb509f86 --- /dev/null +++ b/scripts/atomic/kerneldoc/andnot @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic bitwise AND NOT with ${desc_order} ordering + * @i: ${int} value + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v & ~@i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/cmpxchg b/scripts/atomic/kerneldoc/cmpxchg new file mode 100644 index 0000000000..3bce328f50 --- /dev/null +++ b/scripts/atomic/kerneldoc/cmpxchg @@ -0,0 +1,14 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * @old: ${int} value to compare with + * @new: ${int} value to assign + * + * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: The original value of @v. + */ +EOF diff --git a/scripts/atomic/kerneldoc/dec b/scripts/atomic/kerneldoc/dec new file mode 100644 index 0000000000..bbeecbc4c2 --- /dev/null +++ b/scripts/atomic/kerneldoc/dec @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic decrement with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v - 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/dec_and_test b/scripts/atomic/kerneldoc/dec_and_test new file mode 100644 index 0000000000..71bbd23ce4 --- /dev/null +++ b/scripts/atomic/kerneldoc/dec_and_test @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic decrement and test if zero with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v - 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/dec_if_positive b/scripts/atomic/kerneldoc/dec_if_positive new file mode 100644 index 0000000000..04f1aed3cf --- /dev/null +++ b/scripts/atomic/kerneldoc/dec_if_positive @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic decrement if positive with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * If (@v > 0), atomically updates @v to (@v - 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: The old value of (@v - 1), regardless of whether @v was updated. + */ +EOF diff --git a/scripts/atomic/kerneldoc/dec_unless_positive b/scripts/atomic/kerneldoc/dec_unless_positive new file mode 100644 index 0000000000..ee73612f03 --- /dev/null +++ b/scripts/atomic/kerneldoc/dec_unless_positive @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic decrement unless positive with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * If (@v <= 0), atomically updates @v to (@v - 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if @v was updated, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/inc b/scripts/atomic/kerneldoc/inc new file mode 100644 index 0000000000..9f14f1b3d2 --- /dev/null +++ b/scripts/atomic/kerneldoc/inc @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic increment with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v + 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/inc_and_test b/scripts/atomic/kerneldoc/inc_and_test new file mode 100644 index 0000000000..971694d59b --- /dev/null +++ b/scripts/atomic/kerneldoc/inc_and_test @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic increment and test if zero with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v + 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/inc_not_zero b/scripts/atomic/kerneldoc/inc_not_zero new file mode 100644 index 0000000000..618be08e65 --- /dev/null +++ b/scripts/atomic/kerneldoc/inc_not_zero @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic increment unless zero with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * If (@v != 0), atomically updates @v to (@v + 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if @v was updated, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/inc_unless_negative b/scripts/atomic/kerneldoc/inc_unless_negative new file mode 100644 index 0000000000..597f23d4dc --- /dev/null +++ b/scripts/atomic/kerneldoc/inc_unless_negative @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic increment unless negative with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * If (@v >= 0), atomically updates @v to (@v + 1) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if @v was updated, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/or b/scripts/atomic/kerneldoc/or new file mode 100644 index 0000000000..55b33de504 --- /dev/null +++ b/scripts/atomic/kerneldoc/or @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic bitwise OR with ${desc_order} ordering + * @i: ${int} value + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v | @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/read b/scripts/atomic/kerneldoc/read new file mode 100644 index 0000000000..89fe6147c9 --- /dev/null +++ b/scripts/atomic/kerneldoc/read @@ -0,0 +1,12 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic load with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * + * Atomically loads the value of @v with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: The value loaded from @v. + */ +EOF diff --git a/scripts/atomic/kerneldoc/set b/scripts/atomic/kerneldoc/set new file mode 100644 index 0000000000..e82cb9ebbc --- /dev/null +++ b/scripts/atomic/kerneldoc/set @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic set with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * @i: ${int} value to assign + * + * Atomically sets @v to @i with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: Nothing. + */ +EOF diff --git a/scripts/atomic/kerneldoc/sub b/scripts/atomic/kerneldoc/sub new file mode 100644 index 0000000000..3ba642d044 --- /dev/null +++ b/scripts/atomic/kerneldoc/sub @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic subtract with ${desc_order} ordering + * @i: ${int} value to subtract + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v - @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF diff --git a/scripts/atomic/kerneldoc/sub_and_test b/scripts/atomic/kerneldoc/sub_and_test new file mode 100644 index 0000000000..d3760f7749 --- /dev/null +++ b/scripts/atomic/kerneldoc/sub_and_test @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic subtract and test if zero with ${desc_order} ordering + * @i: ${int} value to add + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v - @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: @true if the resulting value of @v is zero, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/try_cmpxchg b/scripts/atomic/kerneldoc/try_cmpxchg new file mode 100644 index 0000000000..296553206c --- /dev/null +++ b/scripts/atomic/kerneldoc/try_cmpxchg @@ -0,0 +1,15 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic compare and exchange with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * @old: pointer to ${int} value to compare with + * @new: ${int} value to assign + * + * If (@v == @old), atomically updates @v to @new with ${desc_order} ordering. + * Otherwise, updates @old to the current value of @v. + * + * ${desc_noinstr} + * + * Return: @true if the exchange occured, @false otherwise. + */ +EOF diff --git a/scripts/atomic/kerneldoc/xchg b/scripts/atomic/kerneldoc/xchg new file mode 100644 index 0000000000..75f04c085f --- /dev/null +++ b/scripts/atomic/kerneldoc/xchg @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic exchange with ${desc_order} ordering + * @v: pointer to ${atomic}_t + * @new: ${int} value to assign + * + * Atomically updates @v to @new with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * Return: The original value of @v. + */ +EOF diff --git a/scripts/atomic/kerneldoc/xor b/scripts/atomic/kerneldoc/xor new file mode 100644 index 0000000000..8837270f28 --- /dev/null +++ b/scripts/atomic/kerneldoc/xor @@ -0,0 +1,13 @@ +cat <<EOF +/** + * ${class}${atomicname}() - atomic bitwise XOR with ${desc_order} ordering + * @i: ${int} value + * @v: pointer to ${atomic}_t + * + * Atomically updates @v to (@v ^ @i) with ${desc_order} ordering. + * + * ${desc_noinstr} + * + * ${desc_return} + */ +EOF |