summaryrefslogtreecommitdiffstats
path: root/third_party/rust/half/src/binary16/convert.rs
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/rust/half/src/binary16/convert.rs')
-rw-r--r--third_party/rust/half/src/binary16/convert.rs491
1 files changed, 491 insertions, 0 deletions
diff --git a/third_party/rust/half/src/binary16/convert.rs b/third_party/rust/half/src/binary16/convert.rs
new file mode 100644
index 0000000000..c2521d8d26
--- /dev/null
+++ b/third_party/rust/half/src/binary16/convert.rs
@@ -0,0 +1,491 @@
+#![allow(dead_code, unused_imports)]
+
+macro_rules! convert_fn {
+ (fn $name:ident($var:ident : $vartype:ty) -> $restype:ty {
+ if feature("f16c") { $f16c:expr }
+ else { $fallback:expr }}) => {
+ #[inline]
+ pub(crate) fn $name($var: $vartype) -> $restype {
+ // Use CPU feature detection if using std
+ #[cfg(all(
+ feature = "use-intrinsics",
+ feature = "std",
+ any(target_arch = "x86", target_arch = "x86_64"),
+ not(target_feature = "f16c")
+ ))]
+ {
+ if is_x86_feature_detected!("f16c") {
+ $f16c
+ } else {
+ $fallback
+ }
+ }
+ // Use intrinsics directly when a compile target or using no_std
+ #[cfg(all(
+ feature = "use-intrinsics",
+ any(target_arch = "x86", target_arch = "x86_64"),
+ target_feature = "f16c"
+ ))]
+ {
+ $f16c
+ }
+ // Fallback to software
+ #[cfg(any(
+ not(feature = "use-intrinsics"),
+ not(any(target_arch = "x86", target_arch = "x86_64")),
+ all(not(feature = "std"), not(target_feature = "f16c"))
+ ))]
+ {
+ $fallback
+ }
+ }
+ };
+}
+
+convert_fn! {
+ fn f32_to_f16(f: f32) -> u16 {
+ if feature("f16c") {
+ unsafe { x86::f32_to_f16_x86_f16c(f) }
+ } else {
+ f32_to_f16_fallback(f)
+ }
+ }
+}
+
+convert_fn! {
+ fn f64_to_f16(f: f64) -> u16 {
+ if feature("f16c") {
+ unsafe { x86::f32_to_f16_x86_f16c(f as f32) }
+ } else {
+ f64_to_f16_fallback(f)
+ }
+ }
+}
+
+convert_fn! {
+ fn f16_to_f32(i: u16) -> f32 {
+ if feature("f16c") {
+ unsafe { x86::f16_to_f32_x86_f16c(i) }
+ } else {
+ f16_to_f32_fallback(i)
+ }
+ }
+}
+
+convert_fn! {
+ fn f16_to_f64(i: u16) -> f64 {
+ if feature("f16c") {
+ unsafe { x86::f16_to_f32_x86_f16c(i) as f64 }
+ } else {
+ f16_to_f64_fallback(i)
+ }
+ }
+}
+
+// TODO: While SIMD versions are faster, further improvements can be made by doing runtime feature
+// detection once at beginning of convert slice method, rather than per chunk
+
+convert_fn! {
+ fn f32x4_to_f16x4(f: &[f32]) -> [u16; 4] {
+ if feature("f16c") {
+ unsafe { x86::f32x4_to_f16x4_x86_f16c(f) }
+ } else {
+ f32x4_to_f16x4_fallback(f)
+ }
+ }
+}
+
+convert_fn! {
+ fn f16x4_to_f32x4(i: &[u16]) -> [f32; 4] {
+ if feature("f16c") {
+ unsafe { x86::f16x4_to_f32x4_x86_f16c(i) }
+ } else {
+ f16x4_to_f32x4_fallback(i)
+ }
+ }
+}
+
+convert_fn! {
+ fn f64x4_to_f16x4(f: &[f64]) -> [u16; 4] {
+ if feature("f16c") {
+ unsafe { x86::f64x4_to_f16x4_x86_f16c(f) }
+ } else {
+ f64x4_to_f16x4_fallback(f)
+ }
+ }
+}
+
+convert_fn! {
+ fn f16x4_to_f64x4(i: &[u16]) -> [f64; 4] {
+ if feature("f16c") {
+ unsafe { x86::f16x4_to_f64x4_x86_f16c(i) }
+ } else {
+ f16x4_to_f64x4_fallback(i)
+ }
+ }
+}
+
+/////////////// Fallbacks ////////////////
+
+// In the below functions, round to nearest, with ties to even.
+// Let us call the most significant bit that will be shifted out the round_bit.
+//
+// Round up if either
+// a) Removed part > tie.
+// (mantissa & round_bit) != 0 && (mantissa & (round_bit - 1)) != 0
+// b) Removed part == tie, and retained part is odd.
+// (mantissa & round_bit) != 0 && (mantissa & (2 * round_bit)) != 0
+// (If removed part == tie and retained part is even, do not round up.)
+// These two conditions can be combined into one:
+// (mantissa & round_bit) != 0 && (mantissa & ((round_bit - 1) | (2 * round_bit))) != 0
+// which can be simplified into
+// (mantissa & round_bit) != 0 && (mantissa & (3 * round_bit - 1)) != 0
+
+fn f32_to_f16_fallback(value: f32) -> u16 {
+ // Convert to raw bytes
+ let x = value.to_bits();
+
+ // Extract IEEE754 components
+ let sign = x & 0x8000_0000u32;
+ let exp = x & 0x7F80_0000u32;
+ let man = x & 0x007F_FFFFu32;
+
+ // Check for all exponent bits being set, which is Infinity or NaN
+ if exp == 0x7F80_0000u32 {
+ // Set mantissa MSB for NaN (and also keep shifted mantissa bits)
+ let nan_bit = if man == 0 { 0 } else { 0x0200u32 };
+ return ((sign >> 16) | 0x7C00u32 | nan_bit | (man >> 13)) as u16;
+ }
+
+ // The number is normalized, start assembling half precision version
+ let half_sign = sign >> 16;
+ // Unbias the exponent, then bias for half precision
+ let unbiased_exp = ((exp >> 23) as i32) - 127;
+ let half_exp = unbiased_exp + 15;
+
+ // Check for exponent overflow, return +infinity
+ if half_exp >= 0x1F {
+ return (half_sign | 0x7C00u32) as u16;
+ }
+
+ // Check for underflow
+ if half_exp <= 0 {
+ // Check mantissa for what we can do
+ if 14 - half_exp > 24 {
+ // No rounding possibility, so this is a full underflow, return signed zero
+ return half_sign as u16;
+ }
+ // Don't forget about hidden leading mantissa bit when assembling mantissa
+ let man = man | 0x0080_0000u32;
+ let mut half_man = man >> (14 - half_exp);
+ // Check for rounding (see comment above functions)
+ let round_bit = 1 << (13 - half_exp);
+ if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+ half_man += 1;
+ }
+ // No exponent for subnormals
+ return (half_sign | half_man) as u16;
+ }
+
+ // Rebias the exponent
+ let half_exp = (half_exp as u32) << 10;
+ let half_man = man >> 13;
+ // Check for rounding (see comment above functions)
+ let round_bit = 0x0000_1000u32;
+ if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+ // Round it
+ ((half_sign | half_exp | half_man) + 1) as u16
+ } else {
+ (half_sign | half_exp | half_man) as u16
+ }
+}
+
+fn f64_to_f16_fallback(value: f64) -> u16 {
+ // Convert to raw bytes, truncating the last 32-bits of mantissa; that precision will always
+ // be lost on half-precision.
+ let val = value.to_bits();
+ let x = (val >> 32) as u32;
+
+ // Extract IEEE754 components
+ let sign = x & 0x8000_0000u32;
+ let exp = x & 0x7FF0_0000u32;
+ let man = x & 0x000F_FFFFu32;
+
+ // Check for all exponent bits being set, which is Infinity or NaN
+ if exp == 0x7FF0_0000u32 {
+ // Set mantissa MSB for NaN (and also keep shifted mantissa bits).
+ // We also have to check the last 32 bits.
+ let nan_bit = if man == 0 && (val as u32 == 0) {
+ 0
+ } else {
+ 0x0200u32
+ };
+ return ((sign >> 16) | 0x7C00u32 | nan_bit | (man >> 10)) as u16;
+ }
+
+ // The number is normalized, start assembling half precision version
+ let half_sign = sign >> 16;
+ // Unbias the exponent, then bias for half precision
+ let unbiased_exp = ((exp >> 20) as i64) - 1023;
+ let half_exp = unbiased_exp + 15;
+
+ // Check for exponent overflow, return +infinity
+ if half_exp >= 0x1F {
+ return (half_sign | 0x7C00u32) as u16;
+ }
+
+ // Check for underflow
+ if half_exp <= 0 {
+ // Check mantissa for what we can do
+ if 10 - half_exp > 21 {
+ // No rounding possibility, so this is a full underflow, return signed zero
+ return half_sign as u16;
+ }
+ // Don't forget about hidden leading mantissa bit when assembling mantissa
+ let man = man | 0x0010_0000u32;
+ let mut half_man = man >> (11 - half_exp);
+ // Check for rounding (see comment above functions)
+ let round_bit = 1 << (10 - half_exp);
+ if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+ half_man += 1;
+ }
+ // No exponent for subnormals
+ return (half_sign | half_man) as u16;
+ }
+
+ // Rebias the exponent
+ let half_exp = (half_exp as u32) << 10;
+ let half_man = man >> 10;
+ // Check for rounding (see comment above functions)
+ let round_bit = 0x0000_0200u32;
+ if (man & round_bit) != 0 && (man & (3 * round_bit - 1)) != 0 {
+ // Round it
+ ((half_sign | half_exp | half_man) + 1) as u16
+ } else {
+ (half_sign | half_exp | half_man) as u16
+ }
+}
+
+fn f16_to_f32_fallback(i: u16) -> f32 {
+ // Check for signed zero
+ if i & 0x7FFFu16 == 0 {
+ return f32::from_bits((i as u32) << 16);
+ }
+
+ let half_sign = (i & 0x8000u16) as u32;
+ let half_exp = (i & 0x7C00u16) as u32;
+ let half_man = (i & 0x03FFu16) as u32;
+
+ // Check for an infinity or NaN when all exponent bits set
+ if half_exp == 0x7C00u32 {
+ // Check for signed infinity if mantissa is zero
+ if half_man == 0 {
+ return f32::from_bits((half_sign << 16) | 0x7F80_0000u32);
+ } else {
+ // NaN, keep current mantissa but also set most significiant mantissa bit
+ return f32::from_bits((half_sign << 16) | 0x7FC0_0000u32 | (half_man << 13));
+ }
+ }
+
+ // Calculate single-precision components with adjusted exponent
+ let sign = half_sign << 16;
+ // Unbias exponent
+ let unbiased_exp = ((half_exp as i32) >> 10) - 15;
+
+ // Check for subnormals, which will be normalized by adjusting exponent
+ if half_exp == 0 {
+ // Calculate how much to adjust the exponent by
+ let e = (half_man as u16).leading_zeros() - 6;
+
+ // Rebias and adjust exponent
+ let exp = (127 - 15 - e) << 23;
+ let man = (half_man << (14 + e)) & 0x7F_FF_FFu32;
+ return f32::from_bits(sign | exp | man);
+ }
+
+ // Rebias exponent for a normalized normal
+ let exp = ((unbiased_exp + 127) as u32) << 23;
+ let man = (half_man & 0x03FFu32) << 13;
+ f32::from_bits(sign | exp | man)
+}
+
+fn f16_to_f64_fallback(i: u16) -> f64 {
+ // Check for signed zero
+ if i & 0x7FFFu16 == 0 {
+ return f64::from_bits((i as u64) << 48);
+ }
+
+ let half_sign = (i & 0x8000u16) as u64;
+ let half_exp = (i & 0x7C00u16) as u64;
+ let half_man = (i & 0x03FFu16) as u64;
+
+ // Check for an infinity or NaN when all exponent bits set
+ if half_exp == 0x7C00u64 {
+ // Check for signed infinity if mantissa is zero
+ if half_man == 0 {
+ return f64::from_bits((half_sign << 48) | 0x7FF0_0000_0000_0000u64);
+ } else {
+ // NaN, keep current mantissa but also set most significiant mantissa bit
+ return f64::from_bits((half_sign << 48) | 0x7FF8_0000_0000_0000u64 | (half_man << 42));
+ }
+ }
+
+ // Calculate double-precision components with adjusted exponent
+ let sign = half_sign << 48;
+ // Unbias exponent
+ let unbiased_exp = ((half_exp as i64) >> 10) - 15;
+
+ // Check for subnormals, which will be normalized by adjusting exponent
+ if half_exp == 0 {
+ // Calculate how much to adjust the exponent by
+ let e = (half_man as u16).leading_zeros() - 6;
+
+ // Rebias and adjust exponent
+ let exp = ((1023 - 15 - e) as u64) << 52;
+ let man = (half_man << (43 + e)) & 0xF_FFFF_FFFF_FFFFu64;
+ return f64::from_bits(sign | exp | man);
+ }
+
+ // Rebias exponent for a normalized normal
+ let exp = ((unbiased_exp + 1023) as u64) << 52;
+ let man = (half_man & 0x03FFu64) << 42;
+ f64::from_bits(sign | exp | man)
+}
+
+#[inline]
+fn f16x4_to_f32x4_fallback(v: &[u16]) -> [f32; 4] {
+ debug_assert!(v.len() >= 4);
+
+ [
+ f16_to_f32_fallback(v[0]),
+ f16_to_f32_fallback(v[1]),
+ f16_to_f32_fallback(v[2]),
+ f16_to_f32_fallback(v[3]),
+ ]
+}
+
+#[inline]
+fn f32x4_to_f16x4_fallback(v: &[f32]) -> [u16; 4] {
+ debug_assert!(v.len() >= 4);
+
+ [
+ f32_to_f16_fallback(v[0]),
+ f32_to_f16_fallback(v[1]),
+ f32_to_f16_fallback(v[2]),
+ f32_to_f16_fallback(v[3]),
+ ]
+}
+
+#[inline]
+fn f16x4_to_f64x4_fallback(v: &[u16]) -> [f64; 4] {
+ debug_assert!(v.len() >= 4);
+
+ [
+ f16_to_f64_fallback(v[0]),
+ f16_to_f64_fallback(v[1]),
+ f16_to_f64_fallback(v[2]),
+ f16_to_f64_fallback(v[3]),
+ ]
+}
+
+#[inline]
+fn f64x4_to_f16x4_fallback(v: &[f64]) -> [u16; 4] {
+ debug_assert!(v.len() >= 4);
+
+ [
+ f64_to_f16_fallback(v[0]),
+ f64_to_f16_fallback(v[1]),
+ f64_to_f16_fallback(v[2]),
+ f64_to_f16_fallback(v[3]),
+ ]
+}
+
+/////////////// x86/x86_64 f16c ////////////////
+#[cfg(all(
+ feature = "use-intrinsics",
+ any(target_arch = "x86", target_arch = "x86_64")
+))]
+mod x86 {
+ use core::{mem::MaybeUninit, ptr};
+
+ #[cfg(target_arch = "x86")]
+ use core::arch::x86::{__m128, __m128i, _mm_cvtph_ps, _mm_cvtps_ph, _MM_FROUND_TO_NEAREST_INT};
+ #[cfg(target_arch = "x86_64")]
+ use core::arch::x86_64::{
+ __m128, __m128i, _mm_cvtph_ps, _mm_cvtps_ph, _MM_FROUND_TO_NEAREST_INT,
+ };
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f16_to_f32_x86_f16c(i: u16) -> f32 {
+ let mut vec = MaybeUninit::<__m128i>::zeroed();
+ vec.as_mut_ptr().cast::<u16>().write(i);
+ let retval = _mm_cvtph_ps(vec.assume_init());
+ *(&retval as *const __m128).cast()
+ }
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f32_to_f16_x86_f16c(f: f32) -> u16 {
+ let mut vec = MaybeUninit::<__m128>::zeroed();
+ vec.as_mut_ptr().cast::<f32>().write(f);
+ let retval = _mm_cvtps_ph(vec.assume_init(), _MM_FROUND_TO_NEAREST_INT);
+ *(&retval as *const __m128i).cast()
+ }
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f16x4_to_f32x4_x86_f16c(v: &[u16]) -> [f32; 4] {
+ debug_assert!(v.len() >= 4);
+
+ let mut vec = MaybeUninit::<__m128i>::zeroed();
+ ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
+ let retval = _mm_cvtph_ps(vec.assume_init());
+ *(&retval as *const __m128).cast()
+ }
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f32x4_to_f16x4_x86_f16c(v: &[f32]) -> [u16; 4] {
+ debug_assert!(v.len() >= 4);
+
+ let mut vec = MaybeUninit::<__m128>::uninit();
+ ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
+ let retval = _mm_cvtps_ph(vec.assume_init(), _MM_FROUND_TO_NEAREST_INT);
+ *(&retval as *const __m128i).cast()
+ }
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f16x4_to_f64x4_x86_f16c(v: &[u16]) -> [f64; 4] {
+ debug_assert!(v.len() >= 4);
+
+ let mut vec = MaybeUninit::<__m128i>::zeroed();
+ ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
+ let retval = _mm_cvtph_ps(vec.assume_init());
+ let array = *(&retval as *const __m128).cast::<[f32; 4]>();
+ // Let compiler vectorize this regular cast for now.
+ // TODO: investigate auto-detecting sse2/avx convert features
+ [
+ array[0] as f64,
+ array[1] as f64,
+ array[2] as f64,
+ array[3] as f64,
+ ]
+ }
+
+ #[target_feature(enable = "f16c")]
+ #[inline]
+ pub(super) unsafe fn f64x4_to_f16x4_x86_f16c(v: &[f64]) -> [u16; 4] {
+ debug_assert!(v.len() >= 4);
+
+ // Let compiler vectorize this regular cast for now.
+ // TODO: investigate auto-detecting sse2/avx convert features
+ let v = [v[0] as f32, v[1] as f32, v[2] as f32, v[3] as f32];
+
+ let mut vec = MaybeUninit::<__m128>::uninit();
+ ptr::copy_nonoverlapping(v.as_ptr(), vec.as_mut_ptr().cast(), 4);
+ let retval = _mm_cvtps_ph(vec.assume_init(), _MM_FROUND_TO_NEAREST_INT);
+ *(&retval as *const __m128i).cast()
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-12 01:01:38 +0000