diff options
Diffstat (limited to '')
29 files changed, 1106 insertions, 0 deletions
diff --git a/src/runtime/internal/sys/consts.go b/src/runtime/internal/sys/consts.go new file mode 100644 index 0000000..98c0f09 --- /dev/null +++ b/src/runtime/internal/sys/consts.go @@ -0,0 +1,36 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +import ( + "internal/goarch" + "internal/goos" +) + +// AIX requires a larger stack for syscalls. +// The race build also needs more stack. See issue 54291. +// This arithmetic must match that in cmd/internal/objabi/stack.go:stackGuardMultiplier. +const StackGuardMultiplier = 1 + goos.IsAix + isRace + +// DefaultPhysPageSize is the default physical page size. +const DefaultPhysPageSize = goarch.DefaultPhysPageSize + +// PCQuantum is the minimal unit for a program counter (1 on x86, 4 on most other systems). +// The various PC tables record PC deltas pre-divided by PCQuantum. +const PCQuantum = goarch.PCQuantum + +// Int64Align is the required alignment for a 64-bit integer (4 on 32-bit systems, 8 on 64-bit). +const Int64Align = goarch.PtrSize + +// MinFrameSize is the size of the system-reserved words at the bottom +// of a frame (just above the architectural stack pointer). +// It is zero on x86 and PtrSize on most non-x86 (LR-based) systems. +// On PowerPC it is larger, to cover three more reserved words: +// the compiler word, the link editor word, and the TOC save word. +const MinFrameSize = goarch.MinFrameSize + +// StackAlign is the required alignment of the SP register. +// The stack must be at least word aligned, but some architectures require more. +const StackAlign = goarch.StackAlign diff --git a/src/runtime/internal/sys/consts_norace.go b/src/runtime/internal/sys/consts_norace.go new file mode 100644 index 0000000..a9613b8 --- /dev/null +++ b/src/runtime/internal/sys/consts_norace.go @@ -0,0 +1,9 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !race + +package sys + +const isRace = 0 diff --git a/src/runtime/internal/sys/consts_race.go b/src/runtime/internal/sys/consts_race.go new file mode 100644 index 0000000..f824fb3 --- /dev/null +++ b/src/runtime/internal/sys/consts_race.go @@ -0,0 +1,9 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build race + +package sys + +const isRace = 1 diff --git a/src/runtime/internal/sys/intrinsics.go b/src/runtime/internal/sys/intrinsics.go new file mode 100644 index 0000000..e6a3758 --- /dev/null +++ b/src/runtime/internal/sys/intrinsics.go @@ -0,0 +1,208 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +// Copied from math/bits to avoid dependence. + +var deBruijn32tab = [32]byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn32 = 0x077CB531 + +var deBruijn64tab = [64]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +const ntz8tab = "" + + "\x08\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x06\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x07\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x06\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x05\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + + "\x04\x00\x01\x00\x02\x00\x01\x00\x03\x00\x01\x00\x02\x00\x01\x00" + +// TrailingZeros32 returns the number of trailing zero bits in x; the result is 32 for x == 0. +func TrailingZeros32(x uint32) int { + if x == 0 { + return 32 + } + // see comment in TrailingZeros64 + return int(deBruijn32tab[(x&-x)*deBruijn32>>(32-5)]) +} + +// TrailingZeros64 returns the number of trailing zero bits in x; the result is 64 for x == 0. +func TrailingZeros64(x uint64) int { + if x == 0 { + return 64 + } + // If popcount is fast, replace code below with return popcount(^x & (x - 1)). + // + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multiplying by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn (64 bit) constant + // is such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + // (Knuth, volume 4, section 7.3.1) + return int(deBruijn64tab[(x&-x)*deBruijn64>>(64-6)]) +} + +// TrailingZeros8 returns the number of trailing zero bits in x; the result is 8 for x == 0. +func TrailingZeros8(x uint8) int { + return int(ntz8tab[x]) +} + +const len8tab = "" + + "\x00\x01\x02\x02\x03\x03\x03\x03\x04\x04\x04\x04\x04\x04\x04\x04" + + "\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05\x05" + + "\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06" + + "\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06\x06" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + + "\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08\x08" + +// Len64 returns the minimum number of bits required to represent x; the result is 0 for x == 0. +// +// nosplit because this is used in src/runtime/histogram.go, which make run in sensitive contexts. +// +//go:nosplit +func Len64(x uint64) (n int) { + if x >= 1<<32 { + x >>= 32 + n = 32 + } + if x >= 1<<16 { + x >>= 16 + n += 16 + } + if x >= 1<<8 { + x >>= 8 + n += 8 + } + return n + int(len8tab[x]) +} + +// --- OnesCount --- + +const m0 = 0x5555555555555555 // 01010101 ... +const m1 = 0x3333333333333333 // 00110011 ... +const m2 = 0x0f0f0f0f0f0f0f0f // 00001111 ... + +// OnesCount64 returns the number of one bits ("population count") in x. +func OnesCount64(x uint64) int { + // Implementation: Parallel summing of adjacent bits. + // See "Hacker's Delight", Chap. 5: Counting Bits. + // The following pattern shows the general approach: + // + // x = x>>1&(m0&m) + x&(m0&m) + // x = x>>2&(m1&m) + x&(m1&m) + // x = x>>4&(m2&m) + x&(m2&m) + // x = x>>8&(m3&m) + x&(m3&m) + // x = x>>16&(m4&m) + x&(m4&m) + // x = x>>32&(m5&m) + x&(m5&m) + // return int(x) + // + // Masking (& operations) can be left away when there's no + // danger that a field's sum will carry over into the next + // field: Since the result cannot be > 64, 8 bits is enough + // and we can ignore the masks for the shifts by 8 and up. + // Per "Hacker's Delight", the first line can be simplified + // more, but it saves at best one instruction, so we leave + // it alone for clarity. + const m = 1<<64 - 1 + x = x>>1&(m0&m) + x&(m0&m) + x = x>>2&(m1&m) + x&(m1&m) + x = (x>>4 + x) & (m2 & m) + x += x >> 8 + x += x >> 16 + x += x >> 32 + return int(x) & (1<<7 - 1) +} + +// LeadingZeros64 returns the number of leading zero bits in x; the result is 64 for x == 0. +func LeadingZeros64(x uint64) int { return 64 - Len64(x) } + +// LeadingZeros8 returns the number of leading zero bits in x; the result is 8 for x == 0. +func LeadingZeros8(x uint8) int { return 8 - Len8(x) } + +// Len8 returns the minimum number of bits required to represent x; the result is 0 for x == 0. +func Len8(x uint8) int { + return int(len8tab[x]) +} + +// Bswap64 returns its input with byte order reversed +// 0x0102030405060708 -> 0x0807060504030201 +func Bswap64(x uint64) uint64 { + c8 := uint64(0x00ff00ff00ff00ff) + a := x >> 8 & c8 + b := (x & c8) << 8 + x = a | b + c16 := uint64(0x0000ffff0000ffff) + a = x >> 16 & c16 + b = (x & c16) << 16 + x = a | b + c32 := uint64(0x00000000ffffffff) + a = x >> 32 & c32 + b = (x & c32) << 32 + x = a | b + return x +} + +// Bswap32 returns its input with byte order reversed +// 0x01020304 -> 0x04030201 +func Bswap32(x uint32) uint32 { + c8 := uint32(0x00ff00ff) + a := x >> 8 & c8 + b := (x & c8) << 8 + x = a | b + c16 := uint32(0x0000ffff) + a = x >> 16 & c16 + b = (x & c16) << 16 + x = a | b + return x +} + +// Prefetch prefetches data from memory addr to cache +// +// AMD64: Produce PREFETCHT0 instruction +// +// ARM64: Produce PRFM instruction with PLDL1KEEP option +func Prefetch(addr uintptr) {} + +// PrefetchStreamed prefetches data from memory addr, with a hint that this data is being streamed. +// That is, it is likely to be accessed very soon, but only once. If possible, this will avoid polluting the cache. +// +// AMD64: Produce PREFETCHNTA instruction +// +// ARM64: Produce PRFM instruction with PLDL1STRM option +func PrefetchStreamed(addr uintptr) {} diff --git a/src/runtime/internal/sys/intrinsics_test.go b/src/runtime/internal/sys/intrinsics_test.go new file mode 100644 index 0000000..6799885 --- /dev/null +++ b/src/runtime/internal/sys/intrinsics_test.go @@ -0,0 +1,42 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys_test + +import ( + "runtime/internal/sys" + "testing" +) + +func TestTrailingZeros64(t *testing.T) { + for i := 0; i <= 64; i++ { + x := uint64(5) << uint(i) + if got := sys.TrailingZeros64(x); got != i { + t.Errorf("TrailingZeros64(%d)=%d, want %d", x, got, i) + } + } +} +func TestTrailingZeros32(t *testing.T) { + for i := 0; i <= 32; i++ { + x := uint32(5) << uint(i) + if got := sys.TrailingZeros32(x); got != i { + t.Errorf("TrailingZeros32(%d)=%d, want %d", x, got, i) + } + } +} + +func TestBswap64(t *testing.T) { + x := uint64(0x1122334455667788) + y := sys.Bswap64(x) + if y != 0x8877665544332211 { + t.Errorf("Bswap(%x)=%x, want 0x8877665544332211", x, y) + } +} +func TestBswap32(t *testing.T) { + x := uint32(0x11223344) + y := sys.Bswap32(x) + if y != 0x44332211 { + t.Errorf("Bswap(%x)=%x, want 0x44332211", x, y) + } +} diff --git a/src/runtime/internal/sys/nih.go b/src/runtime/internal/sys/nih.go new file mode 100644 index 0000000..17eab67 --- /dev/null +++ b/src/runtime/internal/sys/nih.go @@ -0,0 +1,41 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sys + +// NOTE: keep in sync with cmd/compile/internal/types.CalcSize +// to make the compiler recognize this as an intrinsic type. +type nih struct{} + +// NotInHeap is a type must never be allocated from the GC'd heap or on the stack, +// and is called not-in-heap. +// +// Other types can embed NotInHeap to make it not-in-heap. Specifically, pointers +// to these types must always fail the `runtime.inheap` check. The type may be used +// for global variables, or for objects in unmanaged memory (e.g., allocated with +// `sysAlloc`, `persistentalloc`, r`fixalloc`, or from a manually-managed span). +// +// Specifically: +// +// 1. `new(T)`, `make([]T)`, `append([]T, ...)` and implicit heap +// allocation of T are disallowed. (Though implicit allocations are +// disallowed in the runtime anyway.) +// +// 2. A pointer to a regular type (other than `unsafe.Pointer`) cannot be +// converted to a pointer to a not-in-heap type, even if they have the +// same underlying type. +// +// 3. Any type that containing a not-in-heap type is itself considered as not-in-heap. +// +// - Structs and arrays are not-in-heap if their elements are not-in-heap. +// - Maps and channels contains no-in-heap types are disallowed. +// +// 4. Write barriers on pointers to not-in-heap types can be omitted. +// +// The last point is the real benefit of NotInHeap. The runtime uses +// it for low-level internal structures to avoid memory barriers in the +// scheduler and the memory allocator where they are illegal or simply +// inefficient. This mechanism is reasonably safe and does not compromise +// the readability of the runtime. +type NotInHeap struct{ _ nih } diff --git a/src/runtime/internal/sys/sys.go b/src/runtime/internal/sys/sys.go new file mode 100644 index 0000000..694101d --- /dev/null +++ b/src/runtime/internal/sys/sys.go @@ -0,0 +1,7 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// package sys contains system- and configuration- and architecture-specific +// constants used by the runtime. +package sys diff --git a/src/runtime/internal/syscall/asm_linux_386.s b/src/runtime/internal/syscall/asm_linux_386.s new file mode 100644 index 0000000..15aae4d --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_386.s @@ -0,0 +1,34 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// See ../sys_linux_386.s for the reason why we always use int 0x80 +// instead of the glibc-specific "CALL 0x10(GS)". +#define INVOKE_SYSCALL INT $0x80 + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +// +// Syscall # in AX, args in BX CX DX SI DI BP, return in AX +TEXT ·Syscall6(SB),NOSPLIT,$0-40 + MOVL num+0(FP), AX // syscall entry + MOVL a1+4(FP), BX + MOVL a2+8(FP), CX + MOVL a3+12(FP), DX + MOVL a4+16(FP), SI + MOVL a5+20(FP), DI + MOVL a6+24(FP), BP + INVOKE_SYSCALL + CMPL AX, $0xfffff001 + JLS ok + MOVL $-1, r1+28(FP) + MOVL $0, r2+32(FP) + NEGL AX + MOVL AX, errno+36(FP) + RET +ok: + MOVL AX, r1+28(FP) + MOVL DX, r2+32(FP) + MOVL $0, errno+36(FP) + RET diff --git a/src/runtime/internal/syscall/asm_linux_amd64.s b/src/runtime/internal/syscall/asm_linux_amd64.s new file mode 100644 index 0000000..3740ef1 --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_amd64.s @@ -0,0 +1,47 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +// +// We need to convert to the syscall ABI. +// +// arg | ABIInternal | Syscall +// --------------------------- +// num | AX | AX +// a1 | BX | DI +// a2 | CX | SI +// a3 | DI | DX +// a4 | SI | R10 +// a5 | R8 | R8 +// a6 | R9 | R9 +// +// r1 | AX | AX +// r2 | BX | DX +// err | CX | part of AX +// +// Note that this differs from "standard" ABI convention, which would pass 4th +// arg in CX, not R10. +TEXT ·Syscall6<ABIInternal>(SB),NOSPLIT,$0 + // a6 already in R9. + // a5 already in R8. + MOVQ SI, R10 // a4 + MOVQ DI, DX // a3 + MOVQ CX, SI // a2 + MOVQ BX, DI // a1 + // num already in AX. + SYSCALL + CMPQ AX, $0xfffffffffffff001 + JLS ok + NEGQ AX + MOVQ AX, CX // errno + MOVQ $-1, AX // r1 + MOVQ $0, BX // r2 + RET +ok: + // r1 already in AX. + MOVQ DX, BX // r2 + MOVQ $0, CX // errno + RET diff --git a/src/runtime/internal/syscall/asm_linux_arm.s b/src/runtime/internal/syscall/asm_linux_arm.s new file mode 100644 index 0000000..dbf1826 --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_arm.s @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·Syscall6(SB),NOSPLIT,$0-40 + MOVW num+0(FP), R7 // syscall entry + MOVW a1+4(FP), R0 + MOVW a2+8(FP), R1 + MOVW a3+12(FP), R2 + MOVW a4+16(FP), R3 + MOVW a5+20(FP), R4 + MOVW a6+24(FP), R5 + SWI $0 + MOVW $0xfffff001, R6 + CMP R6, R0 + BLS ok + MOVW $-1, R1 + MOVW R1, r1+28(FP) + MOVW $0, R2 + MOVW R2, r2+32(FP) + RSB $0, R0, R0 + MOVW R0, errno+36(FP) + RET +ok: + MOVW R0, r1+28(FP) + MOVW R1, r2+32(FP) + MOVW $0, R0 + MOVW R0, errno+36(FP) + RET diff --git a/src/runtime/internal/syscall/asm_linux_arm64.s b/src/runtime/internal/syscall/asm_linux_arm64.s new file mode 100644 index 0000000..83e862f --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_arm64.s @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·Syscall6(SB),NOSPLIT,$0-80 + MOVD num+0(FP), R8 // syscall entry + MOVD a1+8(FP), R0 + MOVD a2+16(FP), R1 + MOVD a3+24(FP), R2 + MOVD a4+32(FP), R3 + MOVD a5+40(FP), R4 + MOVD a6+48(FP), R5 + SVC + CMN $4095, R0 + BCC ok + MOVD $-1, R4 + MOVD R4, r1+56(FP) + MOVD ZR, r2+64(FP) + NEG R0, R0 + MOVD R0, errno+72(FP) + RET +ok: + MOVD R0, r1+56(FP) + MOVD R1, r2+64(FP) + MOVD ZR, errno+72(FP) + RET diff --git a/src/runtime/internal/syscall/asm_linux_loong64.s b/src/runtime/internal/syscall/asm_linux_loong64.s new file mode 100644 index 0000000..11c5bc2 --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_loong64.s @@ -0,0 +1,68 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +// +// We need to convert to the syscall ABI. +// +// arg | ABIInternal | Syscall +// --------------------------- +// num | R4 | R11 +// a1 | R5 | R4 +// a2 | R6 | R5 +// a3 | R7 | R6 +// a4 | R8 | R7 +// a5 | R9 | R8 +// a6 | R10 | R9 +// +// r1 | R4 | R4 +// r2 | R5 | R5 +// err | R6 | part of R4 +TEXT ·Syscall6<ABIInternal>(SB),NOSPLIT,$0-80 +#ifdef GOEXPERIMENT_regabiargs + MOVV R4, R11 // syscall entry + MOVV R5, R4 + MOVV R6, R5 + MOVV R7, R6 + MOVV R8, R7 + MOVV R9, R8 + MOVV R10, R9 +#else + MOVV num+0(FP), R11 // syscall entry + MOVV a1+8(FP), R4 + MOVV a2+16(FP), R5 + MOVV a3+24(FP), R6 + MOVV a4+32(FP), R7 + MOVV a5+40(FP), R8 + MOVV a6+48(FP), R9 +#endif + SYSCALL +#ifdef GOEXPERIMENT_regabiargs + MOVV R0, R5 // r2 is not used. Always set to 0. + MOVW $-4096, R12 + BGEU R12, R4, ok + SUBVU R4, R0, R6 // errno + MOVV $-1, R4 // r1 +#else + MOVW $-4096, R12 + BGEU R12, R4, ok + MOVV $-1, R12 + MOVV R12, r1+56(FP) + MOVV R0, r2+64(FP) + SUBVU R4, R0, R4 + MOVV R4, errno+72(FP) +#endif + RET +ok: +#ifdef GOEXPERIMENT_regabiargs + // r1 already in R4 + MOVV R0, R6 // errno +#else + MOVV R4, r1+56(FP) + MOVV R0, r2+64(FP) // r2 is not used. Always set to 0. + MOVV R0, errno+72(FP) +#endif + RET diff --git a/src/runtime/internal/syscall/asm_linux_mips64x.s b/src/runtime/internal/syscall/asm_linux_mips64x.s new file mode 100644 index 0000000..6b7c524 --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_mips64x.s @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips64 || mips64le) + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·Syscall6(SB),NOSPLIT,$0-80 + MOVV num+0(FP), R2 // syscall entry + MOVV a1+8(FP), R4 + MOVV a2+16(FP), R5 + MOVV a3+24(FP), R6 + MOVV a4+32(FP), R7 + MOVV a5+40(FP), R8 + MOVV a6+48(FP), R9 + MOVV R0, R3 // reset R3 to 0 as 1-ret SYSCALL keeps it + SYSCALL + BEQ R7, ok + MOVV $-1, R1 + MOVV R1, r1+56(FP) + MOVV R0, r2+64(FP) + MOVV R2, errno+72(FP) + RET +ok: + MOVV R2, r1+56(FP) + MOVV R3, r2+64(FP) + MOVV R0, errno+72(FP) + RET diff --git a/src/runtime/internal/syscall/asm_linux_mipsx.s b/src/runtime/internal/syscall/asm_linux_mipsx.s new file mode 100644 index 0000000..561310f --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_mipsx.s @@ -0,0 +1,35 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips || mipsle) + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +// +// The 5th and 6th arg go at sp+16, sp+20. +// Note that frame size of 20 means that 24 bytes gets reserved on stack. +TEXT ·Syscall6(SB),NOSPLIT,$20-40 + MOVW num+0(FP), R2 // syscall entry + MOVW a1+4(FP), R4 + MOVW a2+8(FP), R5 + MOVW a3+12(FP), R6 + MOVW a4+16(FP), R7 + MOVW a5+20(FP), R8 + MOVW a6+24(FP), R9 + MOVW R8, 16(R29) + MOVW R9, 20(R29) + MOVW R0, R3 // reset R3 to 0 as 1-ret SYSCALL keeps it + SYSCALL + BEQ R7, ok + MOVW $-1, R1 + MOVW R1, r1+28(FP) + MOVW R0, r2+32(FP) + MOVW R2, errno+36(FP) + RET +ok: + MOVW R2, r1+28(FP) + MOVW R3, r2+32(FP) + MOVW R0, errno+36(FP) + RET diff --git a/src/runtime/internal/syscall/asm_linux_ppc64x.s b/src/runtime/internal/syscall/asm_linux_ppc64x.s new file mode 100644 index 0000000..3e985ed --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_ppc64x.s @@ -0,0 +1,23 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (ppc64 || ppc64le) + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·Syscall6<ABIInternal>(SB),NOSPLIT,$0-80 + MOVD R3, R10 // Move syscall number to R10. SYSCALL will move it R0, and restore R0. + MOVD R4, R3 + MOVD R5, R4 + MOVD R6, R5 + MOVD R7, R6 + MOVD R8, R7 + MOVD R9, R8 + SYSCALL R10 + MOVD $-1, R6 + ISEL CR0SO, R3, R0, R5 // errno = (error) ? R3 : 0 + ISEL CR0SO, R6, R3, R3 // r1 = (error) ? -1 : 0 + MOVD $0, R4 // r2 is not used on linux/ppc64 + RET diff --git a/src/runtime/internal/syscall/asm_linux_riscv64.s b/src/runtime/internal/syscall/asm_linux_riscv64.s new file mode 100644 index 0000000..15e50ec --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_riscv64.s @@ -0,0 +1,43 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +// +// We need to convert to the syscall ABI. +// +// arg | ABIInternal | Syscall +// --------------------------- +// num | A0 | A7 +// a1 | A1 | A0 +// a2 | A2 | A1 +// a3 | A3 | A2 +// a4 | A4 | A3 +// a5 | A5 | A4 +// a6 | A6 | A5 +// +// r1 | A0 | A0 +// r2 | A1 | A1 +// err | A2 | part of A0 +TEXT ·Syscall6<ABIInternal>(SB),NOSPLIT,$0-80 + MOV A0, A7 + MOV A1, A0 + MOV A2, A1 + MOV A3, A2 + MOV A4, A3 + MOV A5, A4 + MOV A6, A5 + ECALL + MOV $-4096, T0 + BLTU T0, A0, err + // r1 already in A0 + // r2 already in A1 + MOV ZERO, A2 // errno + RET +err: + SUB A0, ZERO, A2 // errno + MOV $-1, A0 // r1 + MOV ZERO, A1 // r2 + RET diff --git a/src/runtime/internal/syscall/asm_linux_s390x.s b/src/runtime/internal/syscall/asm_linux_s390x.s new file mode 100644 index 0000000..1b27f29 --- /dev/null +++ b/src/runtime/internal/syscall/asm_linux_s390x.s @@ -0,0 +1,28 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) +TEXT ·Syscall6(SB),NOSPLIT,$0-80 + MOVD num+0(FP), R1 // syscall entry + MOVD a1+8(FP), R2 + MOVD a2+16(FP), R3 + MOVD a3+24(FP), R4 + MOVD a4+32(FP), R5 + MOVD a5+40(FP), R6 + MOVD a6+48(FP), R7 + SYSCALL + MOVD $0xfffffffffffff001, R8 + CMPUBLT R2, R8, ok + MOVD $-1, r1+56(FP) + MOVD $0, r2+64(FP) + NEG R2, R2 + MOVD R2, errno+72(FP) + RET +ok: + MOVD R2, r1+56(FP) + MOVD R3, r2+64(FP) + MOVD $0, errno+72(FP) + RET diff --git a/src/runtime/internal/syscall/defs_linux_386.go b/src/runtime/internal/syscall/defs_linux_386.go new file mode 100644 index 0000000..dc723a6 --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_386.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_FCNTL = 55 + SYS_EPOLL_CTL = 255 + SYS_EPOLL_PWAIT = 319 + SYS_EPOLL_CREATE1 = 329 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + Data [8]byte // to match amd64 +} diff --git a/src/runtime/internal/syscall/defs_linux_amd64.go b/src/runtime/internal/syscall/defs_linux_amd64.go new file mode 100644 index 0000000..886eb5b --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_amd64.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_FCNTL = 72 + SYS_EPOLL_CTL = 233 + SYS_EPOLL_PWAIT = 281 + SYS_EPOLL_CREATE1 = 291 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/defs_linux_arm.go b/src/runtime/internal/syscall/defs_linux_arm.go new file mode 100644 index 0000000..8f812a2 --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_arm.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_FCNTL = 55 + SYS_EPOLL_CTL = 251 + SYS_EPOLL_PWAIT = 346 + SYS_EPOLL_CREATE1 = 357 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + _pad uint32 + Data [8]byte // to match amd64 +} diff --git a/src/runtime/internal/syscall/defs_linux_arm64.go b/src/runtime/internal/syscall/defs_linux_arm64.go new file mode 100644 index 0000000..48e11b0 --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_arm64.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_EPOLL_CREATE1 = 20 + SYS_EPOLL_CTL = 21 + SYS_EPOLL_PWAIT = 22 + SYS_FCNTL = 25 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + _pad uint32 + Data [8]byte // to match amd64 +} diff --git a/src/runtime/internal/syscall/defs_linux_loong64.go b/src/runtime/internal/syscall/defs_linux_loong64.go new file mode 100644 index 0000000..b78ef81 --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_loong64.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_EPOLL_CREATE1 = 20 + SYS_EPOLL_CTL = 21 + SYS_EPOLL_PWAIT = 22 + SYS_FCNTL = 25 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/defs_linux_mips64x.go b/src/runtime/internal/syscall/defs_linux_mips64x.go new file mode 100644 index 0000000..92b49ca --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_mips64x.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips64 || mips64le) + +package syscall + +const ( + SYS_FCNTL = 5070 + SYS_EPOLL_CTL = 5208 + SYS_EPOLL_PWAIT = 5272 + SYS_EPOLL_CREATE1 = 5285 + SYS_EPOLL_PWAIT2 = 5441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/defs_linux_mipsx.go b/src/runtime/internal/syscall/defs_linux_mipsx.go new file mode 100644 index 0000000..e28d09c --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_mipsx.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (mips || mipsle) + +package syscall + +const ( + SYS_FCNTL = 4055 + SYS_EPOLL_CTL = 4249 + SYS_EPOLL_PWAIT = 4313 + SYS_EPOLL_CREATE1 = 4326 + SYS_EPOLL_PWAIT2 = 4441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data uint64 +} diff --git a/src/runtime/internal/syscall/defs_linux_ppc64x.go b/src/runtime/internal/syscall/defs_linux_ppc64x.go new file mode 100644 index 0000000..a74483e --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_ppc64x.go @@ -0,0 +1,32 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (ppc64 || ppc64le) + +package syscall + +const ( + SYS_FCNTL = 55 + SYS_EPOLL_CTL = 237 + SYS_EPOLL_PWAIT = 303 + SYS_EPOLL_CREATE1 = 315 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/defs_linux_riscv64.go b/src/runtime/internal/syscall/defs_linux_riscv64.go new file mode 100644 index 0000000..b78ef81 --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_riscv64.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_EPOLL_CREATE1 = 20 + SYS_EPOLL_CTL = 21 + SYS_EPOLL_PWAIT = 22 + SYS_FCNTL = 25 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/defs_linux_s390x.go b/src/runtime/internal/syscall/defs_linux_s390x.go new file mode 100644 index 0000000..a7bb1ba --- /dev/null +++ b/src/runtime/internal/syscall/defs_linux_s390x.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall + +const ( + SYS_FCNTL = 55 + SYS_EPOLL_CTL = 250 + SYS_EPOLL_PWAIT = 312 + SYS_EPOLL_CREATE1 = 327 + SYS_EPOLL_PWAIT2 = 441 + + EPOLLIN = 0x1 + EPOLLOUT = 0x4 + EPOLLERR = 0x8 + EPOLLHUP = 0x10 + EPOLLRDHUP = 0x2000 + EPOLLET = 0x80000000 + EPOLL_CLOEXEC = 0x80000 + EPOLL_CTL_ADD = 0x1 + EPOLL_CTL_DEL = 0x2 + EPOLL_CTL_MOD = 0x3 +) + +type EpollEvent struct { + Events uint32 + pad_cgo_0 [4]byte + Data [8]byte // unaligned uintptr +} diff --git a/src/runtime/internal/syscall/syscall_linux.go b/src/runtime/internal/syscall/syscall_linux.go new file mode 100644 index 0000000..7209634 --- /dev/null +++ b/src/runtime/internal/syscall/syscall_linux.go @@ -0,0 +1,62 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syscall provides the syscall primitives required for the runtime. +package syscall + +import ( + "unsafe" +) + +// TODO(https://go.dev/issue/51087): This package is incomplete and currently +// only contains very minimal support for Linux. + +// Syscall6 calls system call number 'num' with arguments a1-6. +func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) + +// syscall_RawSyscall6 is a push linkname to export Syscall6 as +// syscall.RawSyscall6. +// +// //go:uintptrkeepalive because the uintptr argument may be converted pointers +// that need to be kept alive in the caller (this is implied for Syscall6 since +// it has no body). +// +// //go:nosplit because stack copying does not account for uintptrkeepalive, so +// the stack must not grow. Stack copying cannot blindly assume that all +// uintptr arguments are pointers, because some values may look like pointers, +// but not really be pointers, and adjusting their value would break the call. +// +// This is a separate wrapper because we can't export one function as two +// names. The assembly implementations name themselves Syscall6 would not be +// affected by a linkname. +// +//go:uintptrkeepalive +//go:nosplit +//go:linkname syscall_RawSyscall6 syscall.RawSyscall6 +func syscall_RawSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) { + return Syscall6(num, a1, a2, a3, a4, a5, a6) +} + +func EpollCreate1(flags int32) (fd int32, errno uintptr) { + r1, _, e := Syscall6(SYS_EPOLL_CREATE1, uintptr(flags), 0, 0, 0, 0, 0) + return int32(r1), e +} + +var _zero uintptr + +func EpollWait(epfd int32, events []EpollEvent, maxev, waitms int32) (n int32, errno uintptr) { + var ev unsafe.Pointer + if len(events) > 0 { + ev = unsafe.Pointer(&events[0]) + } else { + ev = unsafe.Pointer(&_zero) + } + r1, _, e := Syscall6(SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(ev), uintptr(maxev), uintptr(waitms), 0, 0) + return int32(r1), e +} + +func EpollCtl(epfd, op, fd int32, event *EpollEvent) (errno uintptr) { + _, _, e := Syscall6(SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) + return e +} diff --git a/src/runtime/internal/syscall/syscall_linux_test.go b/src/runtime/internal/syscall/syscall_linux_test.go new file mode 100644 index 0000000..1976da5 --- /dev/null +++ b/src/runtime/internal/syscall/syscall_linux_test.go @@ -0,0 +1,19 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syscall_test + +import ( + "runtime/internal/syscall" + "testing" +) + +func TestEpollctlErrorSign(t *testing.T) { + v := syscall.EpollCtl(-1, 1, -1, &syscall.EpollEvent{}) + + const EBADF = 0x09 + if v != EBADF { + t.Errorf("epollctl = %v, want %v", v, EBADF) + } +} |