path: root/library/stdarch/crates/std_detect/src/detect/os/linux/aarch64.rs

//! Run-time feature detection for Aarch64 on Linux.

use super::auxvec;
use crate::detect::{bit, cache, Feature};

/// Try to read the features from the auxiliary vector, and if that fails, try
/// to read them from /proc/cpuinfo.
pub(crate) fn detect_features() -> cache::Initializer {
    if let Ok(auxv) = auxvec::auxv() {
        let hwcap: AtHwcap = auxv.into();
        return hwcap.cache();
    }
    #[cfg(feature = "std_detect_file_io")]
    if let Ok(c) = super::cpuinfo::CpuInfo::new() {
        let hwcap: AtHwcap = c.into();
        return hwcap.cache();
    }
    cache::Initializer::default()
}

/// These values are part of the platform-specific [asm/hwcap.h][hwcap] .
///
/// The names match those used for cpuinfo.
///
/// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h
#[derive(Debug, Default, PartialEq)]
struct AtHwcap {
    // AT_HWCAP
    fp: bool,
    asimd: bool,
    // evtstrm: No LLVM support.
    aes: bool,
    pmull: bool,
    sha1: bool,
    sha2: bool,
    crc32: bool,
    atomics: bool,
    fphp: bool,
    asimdhp: bool,
    // cpuid: No LLVM support.
    asimdrdm: bool,
    jscvt: bool,
    fcma: bool,
    lrcpc: bool,
    dcpop: bool,
    sha3: bool,
    sm3: bool,
    sm4: bool,
    asimddp: bool,
    sha512: bool,
    sve: bool,
    fhm: bool,
    dit: bool,
    uscat: bool,
    ilrcpc: bool,
    flagm: bool,
    ssbs: bool,
    sb: bool,
    paca: bool,
    pacg: bool,

    // AT_HWCAP2
    dcpodp: bool,
    sve2: bool,
    sveaes: bool,
    // svepmull: No LLVM support.
    svebitperm: bool,
    svesha3: bool,
    svesm4: bool,
    // flagm2: No LLVM support.
    frint: bool,
    // svei8mm: See i8mm feature.
    svef32mm: bool,
    svef64mm: bool,
    // svebf16: See bf16 feature.
    i8mm: bool,
    bf16: bool,
    // dgh: No LLVM support.
    rng: bool,
    bti: bool,
    mte: bool,
}

impl From<auxvec::AuxVec> for AtHwcap {
    /// Reads AtHwcap from the auxiliary vector.
    fn from(auxv: auxvec::AuxVec) -> Self {
        AtHwcap {
            fp: bit::test(auxv.hwcap, 0),
            asimd: bit::test(auxv.hwcap, 1),
            // evtstrm: bit::test(auxv.hwcap, 2),
            aes: bit::test(auxv.hwcap, 3),
            pmull: bit::test(auxv.hwcap, 4),
            sha1: bit::test(auxv.hwcap, 5),
            sha2: bit::test(auxv.hwcap, 6),
            crc32: bit::test(auxv.hwcap, 7),
            atomics: bit::test(auxv.hwcap, 8),
            fphp: bit::test(auxv.hwcap, 9),
            asimdhp: bit::test(auxv.hwcap, 10),
            // cpuid: bit::test(auxv.hwcap, 11),
            asimdrdm: bit::test(auxv.hwcap, 12),
            jscvt: bit::test(auxv.hwcap, 13),
            fcma: bit::test(auxv.hwcap, 14),
            lrcpc: bit::test(auxv.hwcap, 15),
            dcpop: bit::test(auxv.hwcap, 16),
            sha3: bit::test(auxv.hwcap, 17),
            sm3: bit::test(auxv.hwcap, 18),
            sm4: bit::test(auxv.hwcap, 19),
            asimddp: bit::test(auxv.hwcap, 20),
            sha512: bit::test(auxv.hwcap, 21),
            sve: bit::test(auxv.hwcap, 22),
            fhm: bit::test(auxv.hwcap, 23),
            dit: bit::test(auxv.hwcap, 24),
            uscat: bit::test(auxv.hwcap, 25),
            ilrcpc: bit::test(auxv.hwcap, 26),
            flagm: bit::test(auxv.hwcap, 27),
            ssbs: bit::test(auxv.hwcap, 28),
            sb: bit::test(auxv.hwcap, 29),
            paca: bit::test(auxv.hwcap, 30),
            pacg: bit::test(auxv.hwcap, 31),
            dcpodp: bit::test(auxv.hwcap2, 0),
            sve2: bit::test(auxv.hwcap2, 1),
            sveaes: bit::test(auxv.hwcap2, 2),
            // svepmull: bit::test(auxv.hwcap2, 3),
            svebitperm: bit::test(auxv.hwcap2, 4),
            svesha3: bit::test(auxv.hwcap2, 5),
            svesm4: bit::test(auxv.hwcap2, 6),
            // flagm2: bit::test(auxv.hwcap2, 7),
            frint: bit::test(auxv.hwcap2, 8),
            // svei8mm: bit::test(auxv.hwcap2, 9),
            svef32mm: bit::test(auxv.hwcap2, 10),
            svef64mm: bit::test(auxv.hwcap2, 11),
            // svebf16: bit::test(auxv.hwcap2, 12),
            i8mm: bit::test(auxv.hwcap2, 13),
            bf16: bit::test(auxv.hwcap2, 14),
            // dgh: bit::test(auxv.hwcap2, 15),
            rng: bit::test(auxv.hwcap2, 16),
            bti: bit::test(auxv.hwcap2, 17),
            mte: bit::test(auxv.hwcap2, 18),
        }
    }
}

#[cfg(feature = "std_detect_file_io")]
impl From<super::cpuinfo::CpuInfo> for AtHwcap {
    /// Reads AtHwcap from /proc/cpuinfo .
    fn from(c: super::cpuinfo::CpuInfo) -> Self {
        let f = &c.field("Features");
        AtHwcap {
            // 64-bit names. FIXME: In 32-bit compatibility mode /proc/cpuinfo will
            // map some of the 64-bit names to some 32-bit feature names. This does not
            // cover that yet.
            fp: f.has("fp"),
            asimd: f.has("asimd"),
            // evtstrm: f.has("evtstrm"),
            aes: f.has("aes"),
            pmull: f.has("pmull"),
            sha1: f.has("sha1"),
            sha2: f.has("sha2"),
            crc32: f.has("crc32"),
            atomics: f.has("atomics"),
            fphp: f.has("fphp"),
            asimdhp: f.has("asimdhp"),
            // cpuid: f.has("cpuid"),
            asimdrdm: f.has("asimdrdm"),
            jscvt: f.has("jscvt"),
            fcma: f.has("fcma"),
            lrcpc: f.has("lrcpc"),
            dcpop: f.has("dcpop"),
            sha3: f.has("sha3"),
            sm3: f.has("sm3"),
            sm4: f.has("sm4"),
            asimddp: f.has("asimddp"),
            sha512: f.has("sha512"),
            sve: f.has("sve"),
            fhm: f.has("asimdfhm"),
            dit: f.has("dit"),
            uscat: f.has("uscat"),
            ilrcpc: f.has("ilrcpc"),
            flagm: f.has("flagm"),
            ssbs: f.has("ssbs"),
            sb: f.has("sb"),
            paca: f.has("paca"),
            pacg: f.has("pacg"),
            dcpodp: f.has("dcpodp"),
            sve2: f.has("sve2"),
            sveaes: f.has("sveaes"),
            // svepmull: f.has("svepmull"),
            svebitperm: f.has("svebitperm"),
            svesha3: f.has("svesha3"),
            svesm4: f.has("svesm4"),
            // flagm2: f.has("flagm2"),
            frint: f.has("frint"),
            // svei8mm: f.has("svei8mm"),
            svef32mm: f.has("svef32mm"),
            svef64mm: f.has("svef64mm"),
            // svebf16: f.has("svebf16"),
            i8mm: f.has("i8mm"),
            bf16: f.has("bf16"),
            // dgh: f.has("dgh"),
            rng: f.has("rng"),
            bti: f.has("bti"),
            mte: f.has("mte"),
        }
    }
}

impl AtHwcap {
    /// Initializes the cache from the feature -bits.
    ///
    /// The feature dependencies here come directly from LLVM's feature definintions:
    /// https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64.td
    fn cache(self) -> cache::Initializer {
        let mut value = cache::Initializer::default();
        {
            let mut enable_feature = |f, enable| {
                if enable {
                    value.set(f as u32);
                }
            };

            enable_feature(Feature::fp, self.fp);
            // Half-float support requires float support
            enable_feature(Feature::fp16, self.fp && self.fphp);
            // FHM (fp16fml in LLVM) requires half float support
            enable_feature(Feature::fhm, self.fphp && self.fhm);
            enable_feature(Feature::pmull, self.pmull);
            enable_feature(Feature::crc, self.crc32);
            enable_feature(Feature::lse, self.atomics);
            enable_feature(Feature::lse2, self.uscat);
            enable_feature(Feature::rcpc, self.lrcpc);
            // RCPC2 (rcpc-immo in LLVM) requires RCPC support
            enable_feature(Feature::rcpc2, self.ilrcpc && self.lrcpc);
            enable_feature(Feature::dit, self.dit);
            enable_feature(Feature::flagm, self.flagm);
            enable_feature(Feature::ssbs, self.ssbs);
            enable_feature(Feature::sb, self.sb);
            enable_feature(Feature::paca, self.paca);
            enable_feature(Feature::pacg, self.pacg);
            enable_feature(Feature::dpb, self.dcpop);
            enable_feature(Feature::dpb2, self.dcpodp);
            enable_feature(Feature::rand, self.rng);
            enable_feature(Feature::bti, self.bti);
            enable_feature(Feature::mte, self.mte);
            // jsconv requires float support
            enable_feature(Feature::jsconv, self.jscvt && self.fp);
            enable_feature(Feature::rdm, self.asimdrdm);
            enable_feature(Feature::dotprod, self.asimddp);
            enable_feature(Feature::frintts, self.frint);

            // FEAT_I8MM & FEAT_BF16 also include optional SVE components which linux exposes
            // separately. We ignore that distinction here.
            enable_feature(Feature::i8mm, self.i8mm);
            enable_feature(Feature::bf16, self.bf16);

            // ASIMD support requires float support - if half-floats are
            // supported, it also requires half-float support:
            let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp);
            enable_feature(Feature::asimd, asimd);
            // ASIMD extensions require ASIMD support:
            enable_feature(Feature::fcma, self.fcma && asimd);
            enable_feature(Feature::sve, self.sve && asimd);

            // SVE extensions require SVE & ASIMD
            enable_feature(Feature::f32mm, self.svef32mm && self.sve && asimd);
            enable_feature(Feature::f64mm, self.svef64mm && self.sve && asimd);

            // Cryptographic extensions require ASIMD
            enable_feature(Feature::aes, self.aes && asimd);
            enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd);
            // SHA512/SHA3 require SHA1 & SHA256
            enable_feature(
                Feature::sha3,
                self.sha512 && self.sha3 && self.sha1 && self.sha2 && asimd,
            );
            enable_feature(Feature::sm4, self.sm3 && self.sm4 && asimd);

            // SVE2 requires SVE
            let sve2 = self.sve2 && self.sve && asimd;
            enable_feature(Feature::sve2, sve2);
            // SVE2 extensions require SVE2 and crypto features
            enable_feature(Feature::sve2_aes, self.sveaes && sve2 && self.aes);
            enable_feature(
                Feature::sve2_sm4,
                self.svesm4 && sve2 && self.sm3 && self.sm4,
            );
            enable_feature(
                Feature::sve2_sha3,
                self.svesha3 && sve2 && self.sha512 && self.sha3 && self.sha1 && self.sha2,
            );
            enable_feature(Feature::sve2_bitperm, self.svebitperm && self.sve2);
        }
        value
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(feature = "std_detect_file_io")]
    mod auxv_from_file {
        use super::auxvec::auxv_from_file;
        use super::*;
        // The baseline hwcaps used in the (artificial) auxv test files.
        fn baseline_hwcaps() -> AtHwcap {
            AtHwcap {
                fp: true,
                asimd: true,
                aes: true,
                pmull: true,
                sha1: true,
                sha2: true,
                crc32: true,
                atomics: true,
                fphp: true,
                asimdhp: true,
                asimdrdm: true,
                lrcpc: true,
                dcpop: true,
                asimddp: true,
                ssbs: true,
                ..AtHwcap::default()
            }
        }

        #[test]
        fn linux_empty_hwcap2_aarch64() {
            let file = concat!(
                env!("CARGO_MANIFEST_DIR"),
                "/src/detect/test_data/linux-empty-hwcap2-aarch64.auxv"
            );
            println!("file: {}", file);
            let v = auxv_from_file(file).unwrap();
            println!("HWCAP : 0x{:0x}", v.hwcap);
            println!("HWCAP2: 0x{:0x}", v.hwcap2);
            assert_eq!(AtHwcap::from(v), baseline_hwcaps());
        }
        #[test]
        fn linux_no_hwcap2_aarch64() {
            let file = concat!(
                env!("CARGO_MANIFEST_DIR"),
                "/src/detect/test_data/linux-no-hwcap2-aarch64.auxv"
            );
            println!("file: {}", file);
            let v = auxv_from_file(file).unwrap();
            println!("HWCAP : 0x{:0x}", v.hwcap);
            println!("HWCAP2: 0x{:0x}", v.hwcap2);
            assert_eq!(AtHwcap::from(v), baseline_hwcaps());
        }
        #[test]
        fn linux_hwcap2_aarch64() {
            let file = concat!(
                env!("CARGO_MANIFEST_DIR"),
                "/src/detect/test_data/linux-hwcap2-aarch64.auxv"
            );
            println!("file: {}", file);
            let v = auxv_from_file(file).unwrap();
            println!("HWCAP : 0x{:0x}", v.hwcap);
            println!("HWCAP2: 0x{:0x}", v.hwcap2);
            assert_eq!(
                AtHwcap::from(v),
                AtHwcap {
                    // Some other HWCAP bits.
                    paca: true,
                    pacg: true,
                    // HWCAP2-only bits.
                    dcpodp: true,
                    frint: true,
                    rng: true,
                    bti: true,
                    mte: true,
                    ..baseline_hwcaps()
                }
            );
        }
    }
}