diff options
Diffstat (limited to 'src/internal/cpu')
32 files changed, 1481 insertions, 0 deletions
diff --git a/src/internal/cpu/cpu.go b/src/internal/cpu/cpu.go new file mode 100644 index 0000000..d794e53 --- /dev/null +++ b/src/internal/cpu/cpu.go @@ -0,0 +1,228 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection +// used by the Go standard library. +package cpu + +// DebugOptions is set to true by the runtime if the OS supports reading +// GODEBUG early in runtime startup. +// This should not be changed after it is initialized. +var DebugOptions bool + +// CacheLinePad is used to pad structs to avoid false sharing. +type CacheLinePad struct{ _ [CacheLinePadSize]byte } + +// CacheLineSize is the CPU's assumed cache line size. +// There is currently no runtime detection of the real cache line size +// so we use the constant per GOARCH CacheLinePadSize as an approximation. +var CacheLineSize uintptr = CacheLinePadSize + +// The booleans in X86 contain the correspondingly named cpuid feature bit. +// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers +// in addition to the cpuid feature bit being set. +// The struct is padded to avoid false sharing. +var X86 struct { + _ CacheLinePad + HasAES bool + HasADX bool + HasAVX bool + HasAVX2 bool + HasAVX512F bool + HasAVX512BW bool + HasAVX512VL bool + HasBMI1 bool + HasBMI2 bool + HasERMS bool + HasFMA bool + HasOSXSAVE bool + HasPCLMULQDQ bool + HasPOPCNT bool + HasRDTSCP bool + HasSHA bool + HasSSE3 bool + HasSSSE3 bool + HasSSE41 bool + HasSSE42 bool + _ CacheLinePad +} + +// The booleans in ARM contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var ARM struct { + _ CacheLinePad + HasVFPv4 bool + HasIDIVA bool + HasV7Atomics bool + _ CacheLinePad +} + +// The booleans in ARM64 contain the correspondingly named cpu feature bit. +// The struct is padded to avoid false sharing. +var ARM64 struct { + _ CacheLinePad + HasAES bool + HasPMULL bool + HasSHA1 bool + HasSHA2 bool + HasSHA512 bool + HasCRC32 bool + HasATOMICS bool + HasCPUID bool + IsNeoverse bool + _ CacheLinePad +} + +var MIPS64X struct { + _ CacheLinePad + HasMSA bool // MIPS SIMD architecture + _ CacheLinePad +} + +// For ppc64(le), it is safe to check only for ISA level starting on ISA v3.00, +// since there are no optional categories. There are some exceptions that also +// require kernel support to work (darn, scv), so there are feature bits for +// those as well. The minimum processor requirement is POWER8 (ISA 2.07). +// The struct is padded to avoid false sharing. +var PPC64 struct { + _ CacheLinePad + HasDARN bool // Hardware random number generator (requires kernel enablement) + HasSCV bool // Syscall vectored (requires kernel enablement) + IsPOWER8 bool // ISA v2.07 (POWER8) + IsPOWER9 bool // ISA v3.00 (POWER9) + IsPOWER10 bool // ISA v3.1 (POWER10) + _ CacheLinePad +} + +var S390X struct { + _ CacheLinePad + HasZARCH bool // z architecture mode is active [mandatory] + HasSTFLE bool // store facility list extended [mandatory] + HasLDISP bool // long (20-bit) displacements [mandatory] + HasEIMM bool // 32-bit immediates [mandatory] + HasDFP bool // decimal floating point + HasETF3EH bool // ETF-3 enhanced + HasMSA bool // message security assist (CPACF) + HasAES bool // KM-AES{128,192,256} functions + HasAESCBC bool // KMC-AES{128,192,256} functions + HasAESCTR bool // KMCTR-AES{128,192,256} functions + HasAESGCM bool // KMA-GCM-AES{128,192,256} functions + HasGHASH bool // KIMD-GHASH function + HasSHA1 bool // K{I,L}MD-SHA-1 functions + HasSHA256 bool // K{I,L}MD-SHA-256 functions + HasSHA512 bool // K{I,L}MD-SHA-512 functions + HasSHA3 bool // K{I,L}MD-SHA3-{224,256,384,512} and K{I,L}MD-SHAKE-{128,256} functions + HasVX bool // vector facility. Note: the runtime sets this when it processes auxv records. + HasVXE bool // vector-enhancements facility 1 + HasKDSA bool // elliptic curve functions + HasECDSA bool // NIST curves + HasEDDSA bool // Edwards curves + _ CacheLinePad +} + +// Initialize examines the processor and sets the relevant variables above. +// This is called by the runtime package early in program initialization, +// before normal init functions are run. env is set by runtime if the OS supports +// cpu feature options in GODEBUG. +func Initialize(env string) { + doinit() + processOptions(env) +} + +// options contains the cpu debug options that can be used in GODEBUG. +// Options are arch dependent and are added by the arch specific doinit functions. +// Features that are mandatory for the specific GOARCH should not be added to options +// (e.g. SSE2 on amd64). +var options []option + +// Option names should be lower case. e.g. avx instead of AVX. +type option struct { + Name string + Feature *bool + Specified bool // whether feature value was specified in GODEBUG + Enable bool // whether feature should be enabled +} + +// processOptions enables or disables CPU feature values based on the parsed env string. +// The env string is expected to be of the form cpu.feature1=value1,cpu.feature2=value2... +// where feature names is one of the architecture specific list stored in the +// cpu packages options variable and values are either 'on' or 'off'. +// If env contains cpu.all=off then all cpu features referenced through the options +// variable are disabled. Other feature names and values result in warning messages. +func processOptions(env string) { +field: + for env != "" { + field := "" + i := indexByte(env, ',') + if i < 0 { + field, env = env, "" + } else { + field, env = env[:i], env[i+1:] + } + if len(field) < 4 || field[:4] != "cpu." { + continue + } + i = indexByte(field, '=') + if i < 0 { + print("GODEBUG: no value specified for \"", field, "\"\n") + continue + } + key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on" + + var enable bool + switch value { + case "on": + enable = true + case "off": + enable = false + default: + print("GODEBUG: value \"", value, "\" not supported for cpu option \"", key, "\"\n") + continue field + } + + if key == "all" { + for i := range options { + options[i].Specified = true + options[i].Enable = enable + } + continue field + } + + for i := range options { + if options[i].Name == key { + options[i].Specified = true + options[i].Enable = enable + continue field + } + } + + print("GODEBUG: unknown cpu feature \"", key, "\"\n") + } + + for _, o := range options { + if !o.Specified { + continue + } + + if o.Enable && !*o.Feature { + print("GODEBUG: can not enable \"", o.Name, "\", missing CPU support\n") + continue + } + + *o.Feature = o.Enable + } +} + +// indexByte returns the index of the first instance of c in s, +// or -1 if c is not present in s. +// indexByte is semantically the same as [strings.IndexByte]. +// We copy this function because "internal/cpu" should not have external dependencies. +func indexByte(s string, c byte) int { + for i := 0; i < len(s); i++ { + if s[i] == c { + return i + } + } + return -1 +} diff --git a/src/internal/cpu/cpu.s b/src/internal/cpu/cpu.s new file mode 100644 index 0000000..3c770c1 --- /dev/null +++ b/src/internal/cpu/cpu.s @@ -0,0 +1,6 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This assembly file exists to allow internal/cpu to call +// non-exported runtime functions that use "go:linkname".
\ No newline at end of file diff --git a/src/internal/cpu/cpu_arm.go b/src/internal/cpu/cpu_arm.go new file mode 100644 index 0000000..080e788 --- /dev/null +++ b/src/internal/cpu/cpu_arm.go @@ -0,0 +1,48 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 32 + +// arm doesn't have a 'cpuid' equivalent, so we rely on HWCAP/HWCAP2. +// These are initialized by archauxv() and should not be changed after they are +// initialized. +var HWCap uint +var HWCap2 uint +var Platform string + +// HWCAP/HWCAP2 bits. These are exposed by Linux and FreeBSD. +const ( + hwcap_VFPv4 = 1 << 16 + hwcap_IDIVA = 1 << 17 + hwcap_LPAE = 1 << 20 +) + +func doinit() { + options = []option{ + {Name: "vfpv4", Feature: &ARM.HasVFPv4}, + {Name: "idiva", Feature: &ARM.HasIDIVA}, + {Name: "v7atomics", Feature: &ARM.HasV7Atomics}, + } + + // HWCAP feature bits + ARM.HasVFPv4 = isSet(HWCap, hwcap_VFPv4) + ARM.HasIDIVA = isSet(HWCap, hwcap_IDIVA) + // lpae is required to make the 64-bit instructions LDRD and STRD (and variants) atomic. + // See ARMv7 manual section B1.6. + // We also need at least a v7 chip, for the DMB instruction. + ARM.HasV7Atomics = isSet(HWCap, hwcap_LPAE) && isV7(Platform) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} + +func isV7(s string) bool { + if s == "aarch64" { + return true + } + return s >= "v7" // will be something like v5, v7, v8, v8l +} diff --git a/src/internal/cpu/cpu_arm64.go b/src/internal/cpu/cpu_arm64.go new file mode 100644 index 0000000..4a302f2 --- /dev/null +++ b/src/internal/cpu/cpu_arm64.go @@ -0,0 +1,69 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +// CacheLinePadSize is used to prevent false sharing of cache lines. +// We choose 128 because Apple Silicon, a.k.a. M1, has 128-byte cache line size. +// It doesn't cost much and is much more future-proof. +const CacheLinePadSize = 128 + +func doinit() { + options = []option{ + {Name: "aes", Feature: &ARM64.HasAES}, + {Name: "pmull", Feature: &ARM64.HasPMULL}, + {Name: "sha1", Feature: &ARM64.HasSHA1}, + {Name: "sha2", Feature: &ARM64.HasSHA2}, + {Name: "sha512", Feature: &ARM64.HasSHA512}, + {Name: "crc32", Feature: &ARM64.HasCRC32}, + {Name: "atomics", Feature: &ARM64.HasATOMICS}, + {Name: "cpuid", Feature: &ARM64.HasCPUID}, + {Name: "isNeoverse", Feature: &ARM64.IsNeoverse}, + } + + // arm64 uses different ways to detect CPU features at runtime depending on the operating system. + osInit() +} + +func getisar0() uint64 + +func getMIDR() uint64 + +func extractBits(data uint64, start, end uint) uint { + return (uint)(data>>start) & ((1 << (end - start + 1)) - 1) +} + +func parseARM64SystemRegisters(isar0 uint64) { + // ID_AA64ISAR0_EL1 + switch extractBits(isar0, 4, 7) { + case 1: + ARM64.HasAES = true + case 2: + ARM64.HasAES = true + ARM64.HasPMULL = true + } + + switch extractBits(isar0, 8, 11) { + case 1: + ARM64.HasSHA1 = true + } + + switch extractBits(isar0, 12, 15) { + case 1: + ARM64.HasSHA2 = true + case 2: + ARM64.HasSHA2 = true + ARM64.HasSHA512 = true + } + + switch extractBits(isar0, 16, 19) { + case 1: + ARM64.HasCRC32 = true + } + + switch extractBits(isar0, 20, 23) { + case 2: + ARM64.HasATOMICS = true + } +} diff --git a/src/internal/cpu/cpu_arm64.s b/src/internal/cpu/cpu_arm64.s new file mode 100644 index 0000000..d6e7f44 --- /dev/null +++ b/src/internal/cpu/cpu_arm64.s @@ -0,0 +1,18 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func getisar0() uint64 +TEXT ·getisar0(SB),NOSPLIT,$0 + // get Instruction Set Attributes 0 into R0 + MRS ID_AA64ISAR0_EL1, R0 + MOVD R0, ret+0(FP) + RET + +// func getMIDR() uint64 +TEXT ·getMIDR(SB), NOSPLIT, $0-8 + MRS MIDR_EL1, R0 + MOVD R0, ret+0(FP) + RET diff --git a/src/internal/cpu/cpu_arm64_android.go b/src/internal/cpu/cpu_arm64_android.go new file mode 100644 index 0000000..fbdf7ba --- /dev/null +++ b/src/internal/cpu/cpu_arm64_android.go @@ -0,0 +1,11 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 + +package cpu + +func osInit() { + hwcapInit("android") +} diff --git a/src/internal/cpu/cpu_arm64_darwin.go b/src/internal/cpu/cpu_arm64_darwin.go new file mode 100644 index 0000000..60beadd --- /dev/null +++ b/src/internal/cpu/cpu_arm64_darwin.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 && darwin && !ios + +package cpu + +func osInit() { + ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00")) + ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00")) + ARM64.HasSHA512 = sysctlEnabled([]byte("hw.optional.armv8_2_sha512\x00")) + + // There are no hw.optional sysctl values for the below features on Mac OS 11.0 + // to detect their supported state dynamically. Assume the CPU features that + // Apple Silicon M1 supports to be available as a minimal set of features + // to all Go programs running on darwin/arm64. + ARM64.HasAES = true + ARM64.HasPMULL = true + ARM64.HasSHA1 = true + ARM64.HasSHA2 = true +} + +//go:noescape +func getsysctlbyname(name []byte) (int32, int32) + +func sysctlEnabled(name []byte) bool { + ret, value := getsysctlbyname(name) + if ret < 0 { + return false + } + return value > 0 +} diff --git a/src/internal/cpu/cpu_arm64_freebsd.go b/src/internal/cpu/cpu_arm64_freebsd.go new file mode 100644 index 0000000..96ed359 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_freebsd.go @@ -0,0 +1,14 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 + +package cpu + +func osInit() { + // Retrieve info from system register ID_AA64ISAR0_EL1. + isar0 := getisar0() + + parseARM64SystemRegisters(isar0) +} diff --git a/src/internal/cpu/cpu_arm64_hwcap.go b/src/internal/cpu/cpu_arm64_hwcap.go new file mode 100644 index 0000000..2fabbb6 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_hwcap.go @@ -0,0 +1,66 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 && linux + +package cpu + +// HWCap may be initialized by archauxv and +// should not be changed after it was initialized. +var HWCap uint + +// HWCAP bits. These are exposed by Linux. +const ( + hwcap_AES = 1 << 3 + hwcap_PMULL = 1 << 4 + hwcap_SHA1 = 1 << 5 + hwcap_SHA2 = 1 << 6 + hwcap_CRC32 = 1 << 7 + hwcap_ATOMICS = 1 << 8 + hwcap_CPUID = 1 << 11 + hwcap_SHA512 = 1 << 21 +) + +func hwcapInit(os string) { + // HWCap was populated by the runtime from the auxiliary vector. + // Use HWCap information since reading aarch64 system registers + // is not supported in user space on older linux kernels. + ARM64.HasAES = isSet(HWCap, hwcap_AES) + ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL) + ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1) + ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2) + ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32) + ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID) + ARM64.HasSHA512 = isSet(HWCap, hwcap_SHA512) + + // The Samsung S9+ kernel reports support for atomics, but not all cores + // actually support them, resulting in SIGILL. See issue #28431. + // TODO(elias.naur): Only disable the optimization on bad chipsets on android. + ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && os != "android" + + // Check to see if executing on a Neoverse core and in order to do that, + // check the AUXV for the CPUID bit. The getMIDR function executes an + // instruction which would normally be an illegal instruction, but it's + // trapped by the kernel, the value sanitized and then returned. + // Without the CPUID bit the kernel will not trap the instruction and the + // process will be terminated with SIGILL. + if ARM64.HasCPUID { + midr := getMIDR() + part_num := uint16((midr >> 4) & 0xfff) + implementor := byte((midr >> 24) & 0xff) + + // d0c - NeoverseN1 + // d40 - NeoverseV1 + // d49 - NeoverseN2 + // d4f - NeoverseV2 + if implementor == 'A' && (part_num == 0xd0c || part_num == 0xd40 || + part_num == 0xd49 || part_num == 0xd4f) { + ARM64.IsNeoverse = true + } + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/src/internal/cpu/cpu_arm64_linux.go b/src/internal/cpu/cpu_arm64_linux.go new file mode 100644 index 0000000..d746bdb --- /dev/null +++ b/src/internal/cpu/cpu_arm64_linux.go @@ -0,0 +1,11 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 && linux && !android + +package cpu + +func osInit() { + hwcapInit("linux") +} diff --git a/src/internal/cpu/cpu_arm64_openbsd.go b/src/internal/cpu/cpu_arm64_openbsd.go new file mode 100644 index 0000000..1259309 --- /dev/null +++ b/src/internal/cpu/cpu_arm64_openbsd.go @@ -0,0 +1,28 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 + +package cpu + +const ( + // From OpenBSD's sys/sysctl.h. + _CTL_MACHDEP = 7 + + // From OpenBSD's machine/cpu.h. + _CPU_ID_AA64ISAR0 = 2 + _CPU_ID_AA64ISAR1 = 3 +) + +//go:noescape +func sysctlUint64(mib []uint32) (uint64, bool) + +func osInit() { + // Get ID_AA64ISAR0 from sysctl. + isar0, ok := sysctlUint64([]uint32{_CTL_MACHDEP, _CPU_ID_AA64ISAR0}) + if !ok { + return + } + parseARM64SystemRegisters(isar0) +} diff --git a/src/internal/cpu/cpu_arm64_other.go b/src/internal/cpu/cpu_arm64_other.go new file mode 100644 index 0000000..44592cf --- /dev/null +++ b/src/internal/cpu/cpu_arm64_other.go @@ -0,0 +1,13 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build arm64 && !linux && !freebsd && !android && (!darwin || ios) && !openbsd + +package cpu + +func osInit() { + // Other operating systems do not support reading HWCap from auxiliary vector, + // reading privileged aarch64 system registers or sysctl in user space to detect + // CPU features at runtime. +} diff --git a/src/internal/cpu/cpu_loong64.go b/src/internal/cpu/cpu_loong64.go new file mode 100644 index 0000000..1c90c24 --- /dev/null +++ b/src/internal/cpu/cpu_loong64.go @@ -0,0 +1,13 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build loong64 + +package cpu + +// CacheLinePadSize is used to prevent false sharing of cache lines. +// We choose 64 because Loongson 3A5000 the L1 Dcache is 4-way 256-line 64-byte-per-line. +const CacheLinePadSize = 64 + +func doinit() {} diff --git a/src/internal/cpu/cpu_mips.go b/src/internal/cpu/cpu_mips.go new file mode 100644 index 0000000..14a9c97 --- /dev/null +++ b/src/internal/cpu/cpu_mips.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 32 + +func doinit() { +} diff --git a/src/internal/cpu/cpu_mips64x.go b/src/internal/cpu/cpu_mips64x.go new file mode 100644 index 0000000..c452ffd --- /dev/null +++ b/src/internal/cpu/cpu_mips64x.go @@ -0,0 +1,32 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips64 || mips64le + +package cpu + +const CacheLinePadSize = 32 + +// This is initialized by archauxv and should not be changed after it is +// initialized. +var HWCap uint + +// HWCAP bits. These are exposed by the Linux kernel 5.4. +const ( + // CPU features + hwcap_MIPS_MSA = 1 << 1 +) + +func doinit() { + options = []option{ + {Name: "msa", Feature: &MIPS64X.HasMSA}, + } + + // HWCAP feature bits + MIPS64X.HasMSA = isSet(HWCap, hwcap_MIPS_MSA) +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/src/internal/cpu/cpu_mipsle.go b/src/internal/cpu/cpu_mipsle.go new file mode 100644 index 0000000..14a9c97 --- /dev/null +++ b/src/internal/cpu/cpu_mipsle.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 32 + +func doinit() { +} diff --git a/src/internal/cpu/cpu_no_name.go b/src/internal/cpu/cpu_no_name.go new file mode 100644 index 0000000..2adfa1b --- /dev/null +++ b/src/internal/cpu/cpu_no_name.go @@ -0,0 +1,18 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !386 && !amd64 && !ppc64 && !ppc64le + +package cpu + +// Name returns the CPU name given by the vendor +// if it can be read directly from memory or by CPU instructions. +// If the CPU name can not be determined an empty string is returned. +// +// Implementations that use the Operating System (e.g. sysctl or /sys/) +// to gather CPU information for display should be placed in internal/sysinfo. +func Name() string { + // "A CPU has no name". + return "" +} diff --git a/src/internal/cpu/cpu_ppc64x.go b/src/internal/cpu/cpu_ppc64x.go new file mode 100644 index 0000000..c4a08fe --- /dev/null +++ b/src/internal/cpu/cpu_ppc64x.go @@ -0,0 +1,35 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le + +package cpu + +const CacheLinePadSize = 128 + +func doinit() { + options = []option{ + {Name: "darn", Feature: &PPC64.HasDARN}, + {Name: "scv", Feature: &PPC64.HasSCV}, + {Name: "power9", Feature: &PPC64.IsPOWER9}, + } + + osinit() +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} + +func Name() string { + switch { + case PPC64.IsPOWER10: + return "POWER10" + case PPC64.IsPOWER9: + return "POWER9" + case PPC64.IsPOWER8: + return "POWER8" + } + return "" +} diff --git a/src/internal/cpu/cpu_ppc64x_aix.go b/src/internal/cpu/cpu_ppc64x_aix.go new file mode 100644 index 0000000..f05ed6f --- /dev/null +++ b/src/internal/cpu/cpu_ppc64x_aix.go @@ -0,0 +1,25 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le + +package cpu + +const ( + // getsystemcfg constants + _SC_IMPL = 2 + _IMPL_POWER8 = 0x10000 + _IMPL_POWER9 = 0x20000 + _IMPL_POWER10 = 0x40000 +) + +func osinit() { + impl := getsystemcfg(_SC_IMPL) + PPC64.IsPOWER8 = isSet(impl, _IMPL_POWER8) + PPC64.IsPOWER9 = isSet(impl, _IMPL_POWER9) + PPC64.IsPOWER10 = isSet(impl, _IMPL_POWER10) +} + +// getsystemcfg is defined in runtime/os2_aix.go +func getsystemcfg(label uint) uint diff --git a/src/internal/cpu/cpu_ppc64x_linux.go b/src/internal/cpu/cpu_ppc64x_linux.go new file mode 100644 index 0000000..9df82ca --- /dev/null +++ b/src/internal/cpu/cpu_ppc64x_linux.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le + +package cpu + +// ppc64 doesn't have a 'cpuid' equivalent, so we rely on HWCAP/HWCAP2. +// These are initialized by archauxv and should not be changed after they are +// initialized. +var HWCap uint +var HWCap2 uint + +// HWCAP bits. These are exposed by Linux. +const ( + // ISA Level + hwcap2_ARCH_2_07 = 0x80000000 + hwcap2_ARCH_3_00 = 0x00800000 + hwcap2_ARCH_3_1 = 0x00040000 + + // CPU features + hwcap2_DARN = 0x00200000 + hwcap2_SCV = 0x00100000 +) + +func osinit() { + PPC64.IsPOWER8 = isSet(HWCap2, hwcap2_ARCH_2_07) + PPC64.IsPOWER9 = isSet(HWCap2, hwcap2_ARCH_3_00) + PPC64.IsPOWER10 = isSet(HWCap2, hwcap2_ARCH_3_1) + PPC64.HasDARN = isSet(HWCap2, hwcap2_DARN) + PPC64.HasSCV = isSet(HWCap2, hwcap2_SCV) +} diff --git a/src/internal/cpu/cpu_ppc64x_other.go b/src/internal/cpu/cpu_ppc64x_other.go new file mode 100644 index 0000000..d5b629d --- /dev/null +++ b/src/internal/cpu/cpu_ppc64x_other.go @@ -0,0 +1,13 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (ppc64 || ppc64le) && !aix && !linux + +package cpu + +func osinit() { + // Other operating systems do not support reading HWCap from auxiliary vector, + // reading privileged system registers or sysctl in user space to detect CPU + // features at runtime. +} diff --git a/src/internal/cpu/cpu_riscv64.go b/src/internal/cpu/cpu_riscv64.go new file mode 100644 index 0000000..2173fe8 --- /dev/null +++ b/src/internal/cpu/cpu_riscv64.go @@ -0,0 +1,10 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 64 + +func doinit() { +} diff --git a/src/internal/cpu/cpu_s390x.go b/src/internal/cpu/cpu_s390x.go new file mode 100644 index 0000000..45d8ed2 --- /dev/null +++ b/src/internal/cpu/cpu_s390x.go @@ -0,0 +1,205 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 256 + +var HWCap uint + +// bitIsSet reports whether the bit at index is set. The bit index +// is in big endian order, so bit index 0 is the leftmost bit. +func bitIsSet(bits []uint64, index uint) bool { + return bits[index/64]&((1<<63)>>(index%64)) != 0 +} + +// function is the function code for the named function. +type function uint8 + +const ( + // KM{,A,C,CTR} function codes + aes128 function = 18 // AES-128 + aes192 function = 19 // AES-192 + aes256 function = 20 // AES-256 + + // K{I,L}MD function codes + sha1 function = 1 // SHA-1 + sha256 function = 2 // SHA-256 + sha512 function = 3 // SHA-512 + sha3_224 function = 32 // SHA3-224 + sha3_256 function = 33 // SHA3-256 + sha3_384 function = 34 // SHA3-384 + sha3_512 function = 35 // SHA3-512 + shake128 function = 36 // SHAKE-128 + shake256 function = 37 // SHAKE-256 + + // KLMD function codes + ghash function = 65 // GHASH +) + +const ( + // KDSA function codes + ecdsaVerifyP256 function = 1 // NIST P256 + ecdsaVerifyP384 function = 2 // NIST P384 + ecdsaVerifyP521 function = 3 // NIST P521 + ecdsaSignP256 function = 9 // NIST P256 + ecdsaSignP384 function = 10 // NIST P384 + ecdsaSignP521 function = 11 // NIST P521 + eddsaVerifyEd25519 function = 32 // Curve25519 + eddsaVerifyEd448 function = 36 // Curve448 + eddsaSignEd25519 function = 40 // Curve25519 + eddsaSignEd448 function = 44 // Curve448 +) + +// queryResult contains the result of a Query function +// call. Bits are numbered in big endian order so the +// leftmost bit (the MSB) is at index 0. +type queryResult struct { + bits [2]uint64 +} + +// Has reports whether the given functions are present. +func (q *queryResult) Has(fns ...function) bool { + if len(fns) == 0 { + panic("no function codes provided") + } + for _, f := range fns { + if !bitIsSet(q.bits[:], uint(f)) { + return false + } + } + return true +} + +// facility is a bit index for the named facility. +type facility uint8 + +const ( + // mandatory facilities + zarch facility = 1 // z architecture mode is active + stflef facility = 7 // store-facility-list-extended + ldisp facility = 18 // long-displacement + eimm facility = 21 // extended-immediate + + // miscellaneous facilities + dfp facility = 42 // decimal-floating-point + etf3eh facility = 30 // extended-translation 3 enhancement + + // cryptography facilities + msa facility = 17 // message-security-assist + msa3 facility = 76 // message-security-assist extension 3 + msa4 facility = 77 // message-security-assist extension 4 + msa5 facility = 57 // message-security-assist extension 5 + msa8 facility = 146 // message-security-assist extension 8 + msa9 facility = 155 // message-security-assist extension 9 + + // vector facilities + vxe facility = 135 // vector-enhancements 1 + + // Note: vx requires kernel support + // and so must be fetched from HWCAP. + + hwcap_VX = 1 << 11 // vector facility +) + +// facilityList contains the result of an STFLE call. +// Bits are numbered in big endian order so the +// leftmost bit (the MSB) is at index 0. +type facilityList struct { + bits [4]uint64 +} + +// Has reports whether the given facilities are present. +func (s *facilityList) Has(fs ...facility) bool { + if len(fs) == 0 { + panic("no facility bits provided") + } + for _, f := range fs { + if !bitIsSet(s.bits[:], uint(f)) { + return false + } + } + return true +} + +// The following feature detection functions are defined in cpu_s390x.s. +// They are likely to be expensive to call so the results should be cached. +func stfle() facilityList +func kmQuery() queryResult +func kmcQuery() queryResult +func kmctrQuery() queryResult +func kmaQuery() queryResult +func kimdQuery() queryResult +func klmdQuery() queryResult +func kdsaQuery() queryResult + +func doinit() { + options = []option{ + {Name: "zarch", Feature: &S390X.HasZARCH}, + {Name: "stfle", Feature: &S390X.HasSTFLE}, + {Name: "ldisp", Feature: &S390X.HasLDISP}, + {Name: "msa", Feature: &S390X.HasMSA}, + {Name: "eimm", Feature: &S390X.HasEIMM}, + {Name: "dfp", Feature: &S390X.HasDFP}, + {Name: "etf3eh", Feature: &S390X.HasETF3EH}, + {Name: "vx", Feature: &S390X.HasVX}, + {Name: "vxe", Feature: &S390X.HasVXE}, + {Name: "kdsa", Feature: &S390X.HasKDSA}, + } + + aes := []function{aes128, aes192, aes256} + facilities := stfle() + + S390X.HasZARCH = facilities.Has(zarch) + S390X.HasSTFLE = facilities.Has(stflef) + S390X.HasLDISP = facilities.Has(ldisp) + S390X.HasEIMM = facilities.Has(eimm) + S390X.HasDFP = facilities.Has(dfp) + S390X.HasETF3EH = facilities.Has(etf3eh) + S390X.HasMSA = facilities.Has(msa) + + if S390X.HasMSA { + // cipher message + km, kmc := kmQuery(), kmcQuery() + S390X.HasAES = km.Has(aes...) + S390X.HasAESCBC = kmc.Has(aes...) + if facilities.Has(msa4) { + kmctr := kmctrQuery() + S390X.HasAESCTR = kmctr.Has(aes...) + } + if facilities.Has(msa8) { + kma := kmaQuery() + S390X.HasAESGCM = kma.Has(aes...) + } + + // compute message digest + kimd := kimdQuery() // intermediate (no padding) + klmd := klmdQuery() // last (padding) + S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1) + S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256) + S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512) + S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist + sha3 := []function{ + sha3_224, sha3_256, sha3_384, sha3_512, + shake128, shake256, + } + S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...) + S390X.HasKDSA = facilities.Has(msa9) // elliptic curves + if S390X.HasKDSA { + kdsa := kdsaQuery() + S390X.HasECDSA = kdsa.Has(ecdsaVerifyP256, ecdsaSignP256, ecdsaVerifyP384, ecdsaSignP384, ecdsaVerifyP521, ecdsaSignP521) + S390X.HasEDDSA = kdsa.Has(eddsaVerifyEd25519, eddsaSignEd25519, eddsaVerifyEd448, eddsaSignEd448) + } + } + + S390X.HasVX = isSet(HWCap, hwcap_VX) + + if S390X.HasVX { + S390X.HasVXE = facilities.Has(vxe) + } +} + +func isSet(hwc uint, value uint) bool { + return hwc&value != 0 +} diff --git a/src/internal/cpu/cpu_s390x.s b/src/internal/cpu/cpu_s390x.s new file mode 100644 index 0000000..4ffbbde --- /dev/null +++ b/src/internal/cpu/cpu_s390x.s @@ -0,0 +1,63 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func stfle() facilityList +TEXT ·stfle(SB), NOSPLIT|NOFRAME, $0-32 + MOVD $ret+0(FP), R1 + MOVD $3, R0 // last doubleword index to store + XC $32, (R1), (R1) // clear 4 doublewords (32 bytes) + WORD $0xb2b01000 // store facility list extended (STFLE) + RET + +// func kmQuery() queryResult +TEXT ·kmQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KM-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KM R2, R4 // cipher message (KM) + RET + +// func kmcQuery() queryResult +TEXT ·kmcQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMC-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KMC R2, R4 // cipher message with chaining (KMC) + RET + +// func kmctrQuery() queryResult +TEXT ·kmctrQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMCTR-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KMCTR R2, R4, R4 // cipher message with counter (KMCTR) + RET + +// func kmaQuery() queryResult +TEXT ·kmaQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KMA-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KMA R2, R6, R4 // cipher message with authentication (KMA) + RET + +// func kimdQuery() queryResult +TEXT ·kimdQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KIMD-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KIMD R2, R4 // compute intermediate message digest (KIMD) + RET + +// func klmdQuery() queryResult +TEXT ·klmdQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KLMD-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KLMD R2, R4 // compute last message digest (KLMD) + RET + +// func kdsaQuery() queryResult +TEXT ·kdsaQuery(SB), NOSPLIT|NOFRAME, $0-16 + MOVD $0, R0 // set function code to 0 (KLMD-Query) + MOVD $ret+0(FP), R1 // address of 16-byte return value + KDSA R0, R4 // compute digital signature authentication + RET + diff --git a/src/internal/cpu/cpu_s390x_test.go b/src/internal/cpu/cpu_s390x_test.go new file mode 100644 index 0000000..ad86858 --- /dev/null +++ b/src/internal/cpu/cpu_s390x_test.go @@ -0,0 +1,63 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu_test + +import ( + "errors" + . "internal/cpu" + "os" + "regexp" + "testing" +) + +func getFeatureList() ([]string, error) { + cpuinfo, err := os.ReadFile("/proc/cpuinfo") + if err != nil { + return nil, err + } + r := regexp.MustCompile("features\\s*:\\s*(.*)") + b := r.FindSubmatch(cpuinfo) + if len(b) < 2 { + return nil, errors.New("no feature list in /proc/cpuinfo") + } + return regexp.MustCompile("\\s+").Split(string(b[1]), -1), nil +} + +func TestS390XAgainstCPUInfo(t *testing.T) { + // mapping of linux feature strings to S390X fields + mapping := make(map[string]*bool) + for _, option := range Options { + mapping[option.Name] = option.Feature + } + + // these must be true on the machines Go supports + mandatory := make(map[string]bool) + mandatory["zarch"] = false + mandatory["eimm"] = false + mandatory["ldisp"] = false + mandatory["stfle"] = false + + features, err := getFeatureList() + if err != nil { + t.Error(err) + } + for _, feature := range features { + if _, ok := mandatory[feature]; ok { + mandatory[feature] = true + } + if flag, ok := mapping[feature]; ok { + if !*flag { + t.Errorf("feature '%v' not detected", feature) + } + } else { + t.Logf("no entry for '%v'", feature) + } + } + for k, v := range mandatory { + if !v { + t.Errorf("mandatory feature '%v' not detected", k) + } + } +} diff --git a/src/internal/cpu/cpu_test.go b/src/internal/cpu/cpu_test.go new file mode 100644 index 0000000..a6fe7f7 --- /dev/null +++ b/src/internal/cpu/cpu_test.go @@ -0,0 +1,61 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu_test + +import ( + . "internal/cpu" + "internal/godebug" + "internal/testenv" + "os" + "os/exec" + "testing" +) + +func MustHaveDebugOptionsSupport(t *testing.T) { + if !DebugOptions { + t.Skipf("skipping test: cpu feature options not supported by OS") + } +} + +func MustSupportFeatureDetection(t *testing.T) { + // TODO: add platforms that do not have CPU feature detection support. +} + +func runDebugOptionsTest(t *testing.T, test string, options string) { + MustHaveDebugOptionsSupport(t) + + testenv.MustHaveExec(t) + + env := "GODEBUG=" + options + + cmd := exec.Command(os.Args[0], "-test.run=^"+test+"$") + cmd.Env = append(cmd.Env, env) + + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s with %s: run failed: %v output:\n%s\n", + test, env, err, string(output)) + } +} + +func TestDisableAllCapabilities(t *testing.T) { + MustSupportFeatureDetection(t) + runDebugOptionsTest(t, "TestAllCapabilitiesDisabled", "cpu.all=off") +} + +func TestAllCapabilitiesDisabled(t *testing.T) { + MustHaveDebugOptionsSupport(t) + + if godebug.New("#cpu.all").Value() != "off" { + t.Skipf("skipping test: GODEBUG=cpu.all=off not set") + } + + for _, o := range Options { + want := false + if got := *o.Feature; got != want { + t.Errorf("%v: expected %v, got %v", o.Name, want, got) + } + } +} diff --git a/src/internal/cpu/cpu_wasm.go b/src/internal/cpu/cpu_wasm.go new file mode 100644 index 0000000..2310ad6 --- /dev/null +++ b/src/internal/cpu/cpu_wasm.go @@ -0,0 +1,10 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 64 + +func doinit() { +} diff --git a/src/internal/cpu/cpu_x86.go b/src/internal/cpu/cpu_x86.go new file mode 100644 index 0000000..f8aa53a --- /dev/null +++ b/src/internal/cpu/cpu_x86.go @@ -0,0 +1,215 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +package cpu + +const CacheLinePadSize = 64 + +// cpuid is implemented in cpu_x86.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s. +func xgetbv() (eax, edx uint32) + +// getGOAMD64level is implemented in cpu_x86.s. Returns number in [1,4]. +func getGOAMD64level() int32 + +const ( + // edx bits + cpuid_SSE2 = 1 << 26 + + // ecx bits + cpuid_SSE3 = 1 << 0 + cpuid_PCLMULQDQ = 1 << 1 + cpuid_SSSE3 = 1 << 9 + cpuid_FMA = 1 << 12 + cpuid_SSE41 = 1 << 19 + cpuid_SSE42 = 1 << 20 + cpuid_POPCNT = 1 << 23 + cpuid_AES = 1 << 25 + cpuid_OSXSAVE = 1 << 27 + cpuid_AVX = 1 << 28 + + // ebx bits + cpuid_BMI1 = 1 << 3 + cpuid_AVX2 = 1 << 5 + cpuid_BMI2 = 1 << 8 + cpuid_ERMS = 1 << 9 + cpuid_AVX512F = 1 << 16 + cpuid_ADX = 1 << 19 + cpuid_SHA = 1 << 29 + cpuid_AVX512BW = 1 << 30 + cpuid_AVX512VL = 1 << 31 + + // edx bits for CPUID 0x80000001 + cpuid_RDTSCP = 1 << 27 +) + +var maxExtendedFunctionInformation uint32 + +func doinit() { + options = []option{ + {Name: "adx", Feature: &X86.HasADX}, + {Name: "aes", Feature: &X86.HasAES}, + {Name: "erms", Feature: &X86.HasERMS}, + {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, + {Name: "rdtscp", Feature: &X86.HasRDTSCP}, + {Name: "sha", Feature: &X86.HasSHA}, + } + level := getGOAMD64level() + if level < 2 { + // These options are required at level 2. At lower levels + // they can be turned off. + options = append(options, + option{Name: "popcnt", Feature: &X86.HasPOPCNT}, + option{Name: "sse3", Feature: &X86.HasSSE3}, + option{Name: "sse41", Feature: &X86.HasSSE41}, + option{Name: "sse42", Feature: &X86.HasSSE42}, + option{Name: "ssse3", Feature: &X86.HasSSSE3}) + } + if level < 3 { + // These options are required at level 3. At lower levels + // they can be turned off. + options = append(options, + option{Name: "avx", Feature: &X86.HasAVX}, + option{Name: "avx2", Feature: &X86.HasAVX2}, + option{Name: "bmi1", Feature: &X86.HasBMI1}, + option{Name: "bmi2", Feature: &X86.HasBMI2}, + option{Name: "fma", Feature: &X86.HasFMA}) + } + if level < 4 { + // These options are required at level 4. At lower levels + // they can be turned off. + options = append(options, + option{Name: "avx512f", Feature: &X86.HasAVX512F}, + option{Name: "avx512bw", Feature: &X86.HasAVX512BW}, + option{Name: "avx512vl", Feature: &X86.HasAVX512VL}, + ) + } + + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + maxExtendedFunctionInformation, _, _, _ = cpuid(0x80000000, 0) + + _, _, ecx1, _ := cpuid(1, 0) + + X86.HasSSE3 = isSet(ecx1, cpuid_SSE3) + X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ) + X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3) + X86.HasSSE41 = isSet(ecx1, cpuid_SSE41) + X86.HasSSE42 = isSet(ecx1, cpuid_SSE42) + X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT) + X86.HasAES = isSet(ecx1, cpuid_AES) + + // OSXSAVE can be false when using older Operating Systems + // or when explicitly disabled on newer Operating Systems by + // e.g. setting the xsavedisable boot option on Windows 10. + X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE) + + // The FMA instruction set extension only has VEX prefixed instructions. + // VEX prefixed instructions require OSXSAVE to be enabled. + // See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2 + // Section 2.4 "AVX and SSE Instruction Exception Specification" + X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasOSXSAVE + + osSupportsAVX := false + osSupportsAVX512 := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2) + + // AVX512 detection does not work on Darwin, + // see https://github.com/golang/go/issues/49233 + // + // Check if opmask, ZMMhi256 and Hi16_ZMM have OS support. + osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7) + } + + X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) + X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX + X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) + X86.HasERMS = isSet(ebx7, cpuid_ERMS) + X86.HasADX = isSet(ebx7, cpuid_ADX) + X86.HasSHA = isSet(ebx7, cpuid_SHA) + + X86.HasAVX512F = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 + if X86.HasAVX512F { + X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW) + X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) + } + + var maxExtendedInformation uint32 + maxExtendedInformation, _, _, _ = cpuid(0x80000000, 0) + + if maxExtendedInformation < 0x80000001 { + return + } + + _, _, _, edxExt1 := cpuid(0x80000001, 0) + X86.HasRDTSCP = isSet(edxExt1, cpuid_RDTSCP) +} + +func isSet(hwc uint32, value uint32) bool { + return hwc&value != 0 +} + +// Name returns the CPU name given by the vendor. +// If the CPU name can not be determined an +// empty string is returned. +func Name() string { + if maxExtendedFunctionInformation < 0x80000004 { + return "" + } + + data := make([]byte, 0, 3*4*4) + + var eax, ebx, ecx, edx uint32 + eax, ebx, ecx, edx = cpuid(0x80000002, 0) + data = appendBytes(data, eax, ebx, ecx, edx) + eax, ebx, ecx, edx = cpuid(0x80000003, 0) + data = appendBytes(data, eax, ebx, ecx, edx) + eax, ebx, ecx, edx = cpuid(0x80000004, 0) + data = appendBytes(data, eax, ebx, ecx, edx) + + // Trim leading spaces. + for len(data) > 0 && data[0] == ' ' { + data = data[1:] + } + + // Trim tail after and including the first null byte. + for i, c := range data { + if c == '\x00' { + data = data[:i] + break + } + } + + return string(data) +} + +func appendBytes(b []byte, args ...uint32) []byte { + for _, arg := range args { + b = append(b, + byte((arg >> 0)), + byte((arg >> 8)), + byte((arg >> 16)), + byte((arg >> 24))) + } + return b +} diff --git a/src/internal/cpu/cpu_x86.s b/src/internal/cpu/cpu_x86.s new file mode 100644 index 0000000..2ee8eca --- /dev/null +++ b/src/internal/cpu/cpu_x86.s @@ -0,0 +1,43 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +#include "textflag.h" + +// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL eaxArg+0(FP), AX + MOVL ecxArg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv() (eax, edx uint32) +TEXT ·xgetbv(SB),NOSPLIT,$0-8 + MOVL $0, CX + XGETBV + MOVL AX, eax+0(FP) + MOVL DX, edx+4(FP) + RET + +// func getGOAMD64level() int32 +TEXT ·getGOAMD64level(SB),NOSPLIT,$0-4 +#ifdef GOAMD64_v4 + MOVL $4, ret+0(FP) +#else +#ifdef GOAMD64_v3 + MOVL $3, ret+0(FP) +#else +#ifdef GOAMD64_v2 + MOVL $2, ret+0(FP) +#else + MOVL $1, ret+0(FP) +#endif +#endif +#endif + RET diff --git a/src/internal/cpu/cpu_x86_test.go b/src/internal/cpu/cpu_x86_test.go new file mode 100644 index 0000000..cc6552b --- /dev/null +++ b/src/internal/cpu/cpu_x86_test.go @@ -0,0 +1,57 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +package cpu_test + +import ( + . "internal/cpu" + "internal/godebug" + "testing" +) + +func TestX86ifAVX2hasAVX(t *testing.T) { + if X86.HasAVX2 && !X86.HasAVX { + t.Fatalf("HasAVX expected true when HasAVX2 is true, got false") + } +} + +func TestX86ifAVX512FhasAVX2(t *testing.T) { + if X86.HasAVX512F && !X86.HasAVX2 { + t.Fatalf("HasAVX2 expected true when HasAVX512F is true, got false") + } +} + +func TestX86ifAVX512BWhasAVX512F(t *testing.T) { + if X86.HasAVX512BW && !X86.HasAVX512F { + t.Fatalf("HasAVX512F expected true when HasAVX512BW is true, got false") + } +} + +func TestX86ifAVX512VLhasAVX512F(t *testing.T) { + if X86.HasAVX512VL && !X86.HasAVX512F { + t.Fatalf("HasAVX512F expected true when HasAVX512VL is true, got false") + } +} + +func TestDisableSSE3(t *testing.T) { + if GetGOAMD64level() > 1 { + t.Skip("skipping test: can't run on GOAMD64>v1 machines") + } + runDebugOptionsTest(t, "TestSSE3DebugOption", "cpu.sse3=off") +} + +func TestSSE3DebugOption(t *testing.T) { + MustHaveDebugOptionsSupport(t) + + if godebug.New("#cpu.sse3").Value() != "off" { + t.Skipf("skipping test: GODEBUG=cpu.sse3=off not set") + } + + want := false + if got := X86.HasSSE3; got != want { + t.Errorf("X86.HasSSE3 expected %v, got %v", want, got) + } +} diff --git a/src/internal/cpu/export_test.go b/src/internal/cpu/export_test.go new file mode 100644 index 0000000..91bfc1b --- /dev/null +++ b/src/internal/cpu/export_test.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +var ( + Options = options +) diff --git a/src/internal/cpu/export_x86_test.go b/src/internal/cpu/export_x86_test.go new file mode 100644 index 0000000..a12b6f2 --- /dev/null +++ b/src/internal/cpu/export_x86_test.go @@ -0,0 +1,11 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 || amd64 + +package cpu + +var ( + GetGOAMD64level = getGOAMD64level +) |