1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
//! Run-time feature detection for Aarch64 on any OS that emulates the mrs instruction.
//!
//! On FreeBSD >= 12.0, Linux >= 4.11 and other operating systems, it is possible to use
//! privileged system registers from userspace to check CPU feature support.
//!
//! AArch64 system registers ID_AA64ISAR0_EL1, ID_AA64PFR0_EL1, ID_AA64ISAR1_EL1
//! have bits dedicated to features like AdvSIMD, CRC32, AES, atomics (LSE), etc.
//! Each part of the register indicates the level of support for a certain feature, e.g.
//! when ID_AA64ISAR0_EL1\[7:4\] is >= 1, AES is supported; when it's >= 2, PMULL is supported.
//!
//! For proper support of [SoCs where different cores have different capabilities](https://medium.com/@jadr2ddude/a-big-little-problem-a-tale-of-big-little-gone-wrong-e7778ce744bb),
//! the OS has to always report only the features supported by all cores, like [FreeBSD does](https://reviews.freebsd.org/D17137#393947).
//!
//! References:
//!
//! - [Zircon implementation](https://fuchsia.googlesource.com/zircon/+/master/kernel/arch/arm64/feature.cpp)
//! - [Linux documentation](https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt)
//! - [ARM documentation](https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers?lang=en)
use crate::detect::{cache, Feature};
use core::arch::asm;
/// Try to read the features from the system registers.
///
/// This will cause SIGILL if the current OS is not trapping the mrs instruction.
pub(crate) fn detect_features() -> cache::Initializer {
// ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0
let aa64isar0: u64;
unsafe {
asm!(
"mrs {}, ID_AA64ISAR0_EL1",
out(reg) aa64isar0,
options(pure, nomem, preserves_flags, nostack)
);
}
// ID_AA64ISAR1_EL1 - Instruction Set Attribute Register 1
let aa64isar1: u64;
unsafe {
asm!(
"mrs {}, ID_AA64ISAR1_EL1",
out(reg) aa64isar1,
options(pure, nomem, preserves_flags, nostack)
);
}
// ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2
let aa64mmfr2: u64;
unsafe {
asm!(
"mrs {}, ID_AA64MMFR2_EL1",
out(reg) aa64mmfr2,
options(pure, nomem, preserves_flags, nostack)
);
}
// ID_AA64PFR0_EL1 - Processor Feature Register 0
let aa64pfr0: u64;
unsafe {
asm!(
"mrs {}, ID_AA64PFR0_EL1",
out(reg) aa64pfr0,
options(pure, nomem, preserves_flags, nostack)
);
}
parse_system_registers(aa64isar0, aa64isar1, aa64mmfr2, Some(aa64pfr0))
}
pub(crate) fn parse_system_registers(
aa64isar0: u64,
aa64isar1: u64,
aa64mmfr2: u64,
aa64pfr0: Option<u64>,
) -> cache::Initializer {
let mut value = cache::Initializer::default();
let mut enable_feature = |f, enable| {
if enable {
value.set(f as u32);
}
};
// ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0
enable_feature(Feature::pmull, bits_shift(aa64isar0, 7, 4) >= 2);
enable_feature(Feature::tme, bits_shift(aa64isar0, 27, 24) == 1);
enable_feature(Feature::lse, bits_shift(aa64isar0, 23, 20) >= 2);
enable_feature(Feature::crc, bits_shift(aa64isar0, 19, 16) >= 1);
// ID_AA64PFR0_EL1 - Processor Feature Register 0
if let Some(aa64pfr0) = aa64pfr0 {
let fp = bits_shift(aa64pfr0, 19, 16) < 0xF;
let fphp = bits_shift(aa64pfr0, 19, 16) >= 1;
let asimd = bits_shift(aa64pfr0, 23, 20) < 0xF;
let asimdhp = bits_shift(aa64pfr0, 23, 20) >= 1;
enable_feature(Feature::fp, fp);
enable_feature(Feature::fp16, fphp);
// SIMD support requires float support - if half-floats are
// supported, it also requires half-float support:
enable_feature(Feature::asimd, fp && asimd && (!fphp | asimdhp));
// SIMD extensions require SIMD support:
enable_feature(Feature::aes, asimd && bits_shift(aa64isar0, 7, 4) >= 1);
let sha1 = bits_shift(aa64isar0, 11, 8) >= 1;
let sha2 = bits_shift(aa64isar0, 15, 12) >= 1;
enable_feature(Feature::sha2, asimd && sha1 && sha2);
enable_feature(Feature::rdm, asimd && bits_shift(aa64isar0, 31, 28) >= 1);
enable_feature(
Feature::dotprod,
asimd && bits_shift(aa64isar0, 47, 44) >= 1,
);
enable_feature(Feature::sve, asimd && bits_shift(aa64pfr0, 35, 32) >= 1);
}
// ID_AA64ISAR1_EL1 - Instruction Set Attribute Register 1
// Check for either APA or API field
enable_feature(Feature::paca, bits_shift(aa64isar1, 11, 4) >= 1);
enable_feature(Feature::rcpc, bits_shift(aa64isar1, 23, 20) >= 1);
// Check for either GPA or GPI field
enable_feature(Feature::pacg, bits_shift(aa64isar1, 31, 24) >= 1);
// ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2
enable_feature(Feature::lse2, bits_shift(aa64mmfr2, 35, 32) >= 1);
value
}
#[inline]
fn bits_shift(x: u64, high: usize, low: usize) -> u64 {
(x >> low) & ((1 << (high - low + 1)) - 1)
}
|
