diff options
Diffstat (limited to 'src/runtime/stubs.go')
-rw-r--r-- | src/runtime/stubs.go | 499 |
1 files changed, 499 insertions, 0 deletions
diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go new file mode 100644 index 0000000..65b7299 --- /dev/null +++ b/src/runtime/stubs.go @@ -0,0 +1,499 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "internal/abi" + "internal/goarch" + "runtime/internal/math" + "unsafe" +) + +// Should be a built-in for unsafe.Pointer? +// +//go:nosplit +func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { + return unsafe.Pointer(uintptr(p) + x) +} + +// getg returns the pointer to the current g. +// The compiler rewrites calls to this function into instructions +// that fetch the g directly (from TLS or from the dedicated register). +func getg() *g + +// mcall switches from the g to the g0 stack and invokes fn(g), +// where g is the goroutine that made the call. +// mcall saves g's current PC/SP in g->sched so that it can be restored later. +// It is up to fn to arrange for that later execution, typically by recording +// g in a data structure, causing something to call ready(g) later. +// mcall returns to the original goroutine g later, when g has been rescheduled. +// fn must not return at all; typically it ends by calling schedule, to let the m +// run other goroutines. +// +// mcall can only be called from g stacks (not g0, not gsignal). +// +// This must NOT be go:noescape: if fn is a stack-allocated closure, +// fn puts g on a run queue, and g executes before fn returns, the +// closure will be invalidated while it is still executing. +func mcall(fn func(*g)) + +// systemstack runs fn on a system stack. +// If systemstack is called from the per-OS-thread (g0) stack, or +// if systemstack is called from the signal handling (gsignal) stack, +// systemstack calls fn directly and returns. +// Otherwise, systemstack is being called from the limited stack +// of an ordinary goroutine. In this case, systemstack switches +// to the per-OS-thread stack, calls fn, and switches back. +// It is common to use a func literal as the argument, in order +// to share inputs and outputs with the code around the call +// to system stack: +// +// ... set up y ... +// systemstack(func() { +// x = bigcall(y) +// }) +// ... use x ... +// +//go:noescape +func systemstack(fn func()) + +//go:nosplit +//go:nowritebarrierrec +func badsystemstack() { + writeErrStr("fatal: systemstack called from unexpected goroutine") +} + +// memclrNoHeapPointers clears n bytes starting at ptr. +// +// Usually you should use typedmemclr. memclrNoHeapPointers should be +// used only when the caller knows that *ptr contains no heap pointers +// because either: +// +// *ptr is initialized memory and its type is pointer-free, or +// +// *ptr is uninitialized memory (e.g., memory that's being reused +// for a new allocation) and hence contains only "junk". +// +// memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n +// is a multiple of the pointer size, then any pointer-aligned, +// pointer-sized portion is cleared atomically. Despite the function +// name, this is necessary because this function is the underlying +// implementation of typedmemclr and memclrHasPointers. See the doc of +// memmove for more details. +// +// The (CPU-specific) implementations of this function are in memclr_*.s. +// +//go:noescape +func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) + +//go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers +func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { + memclrNoHeapPointers(ptr, n) +} + +// memmove copies n bytes from "from" to "to". +// +// memmove ensures that any pointer in "from" is written to "to" with +// an indivisible write, so that racy reads cannot observe a +// half-written pointer. This is necessary to prevent the garbage +// collector from observing invalid pointers, and differs from memmove +// in unmanaged languages. However, memmove is only required to do +// this if "from" and "to" may contain pointers, which can only be the +// case if "from", "to", and "n" are all be word-aligned. +// +// Implementations are in memmove_*.s. +// +//go:noescape +func memmove(to, from unsafe.Pointer, n uintptr) + +// Outside assembly calls memmove. Make sure it has ABI wrappers. +// +//go:linkname memmove + +//go:linkname reflect_memmove reflect.memmove +func reflect_memmove(to, from unsafe.Pointer, n uintptr) { + memmove(to, from, n) +} + +// exported value for testing +const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) + +//go:nosplit +func fastrand() uint32 { + mp := getg().m + // Implement wyrand: https://github.com/wangyi-fudan/wyhash + // Only the platform that math.Mul64 can be lowered + // by the compiler should be in this list. + if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| + goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| + goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 { + mp.fastrand += 0xa0761d6478bd642f + hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) + return uint32(hi ^ lo) + } + + // Implement xorshift64+: 2 32-bit xorshift sequences added together. + // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's + // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf + // This generator passes the SmallCrush suite, part of TestU01 framework: + // http://simul.iro.umontreal.ca/testu01/tu01.html + t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) + s1, s0 := t[0], t[1] + s1 ^= s1 << 17 + s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 + t[0], t[1] = s0, s1 + return s0 + s1 +} + +//go:nosplit +func fastrandn(n uint32) uint32 { + // This is similar to fastrand() % n, but faster. + // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32(uint64(fastrand()) * uint64(n) >> 32) +} + +func fastrand64() uint64 { + mp := getg().m + // Implement wyrand: https://github.com/wangyi-fudan/wyhash + // Only the platform that math.Mul64 can be lowered + // by the compiler should be in this list. + if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| + goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| + goarch.IsS390x|goarch.IsRiscv64 == 1 { + mp.fastrand += 0xa0761d6478bd642f + hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) + return hi ^ lo + } + + // Implement xorshift64+: 2 32-bit xorshift sequences added together. + // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf + // This generator passes the SmallCrush suite, part of TestU01 framework: + // http://simul.iro.umontreal.ca/testu01/tu01.html + t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) + s1, s0 := t[0], t[1] + s1 ^= s1 << 17 + s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 + r := uint64(s0 + s1) + + s0, s1 = s1, s0 + s1 ^= s1 << 17 + s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 + r += uint64(s0+s1) << 32 + + t[0], t[1] = s0, s1 + return r +} + +func fastrandu() uint { + if goarch.PtrSize == 4 { + return uint(fastrand()) + } + return uint(fastrand64()) +} + +//go:linkname rand_fastrand64 math/rand.fastrand64 +func rand_fastrand64() uint64 { return fastrand64() } + +//go:linkname sync_fastrandn sync.fastrandn +func sync_fastrandn(n uint32) uint32 { return fastrandn(n) } + +//go:linkname net_fastrandu net.fastrandu +func net_fastrandu() uint { return fastrandu() } + +//go:linkname os_fastrand os.fastrand +func os_fastrand() uint32 { return fastrand() } + +// in internal/bytealg/equal_*.s +// +//go:noescape +func memequal(a, b unsafe.Pointer, size uintptr) bool + +// noescape hides a pointer from escape analysis. noescape is +// the identity function but escape analysis doesn't think the +// output depends on the input. noescape is inlined and currently +// compiles down to zero instructions. +// USE CAREFULLY! +// +//go:nosplit +func noescape(p unsafe.Pointer) unsafe.Pointer { + x := uintptr(p) + return unsafe.Pointer(x ^ 0) +} + +// noEscapePtr hides a pointer from escape analysis. See noescape. +// USE CAREFULLY! +// +//go:nosplit +func noEscapePtr[T any](p *T) *T { + x := uintptr(unsafe.Pointer(p)) + return (*T)(unsafe.Pointer(x ^ 0)) +} + +// Not all cgocallback frames are actually cgocallback, +// so not all have these arguments. Mark them uintptr so that the GC +// does not misinterpret memory when the arguments are not present. +// cgocallback is not called from Go, only from crosscall2. +// This in turn calls cgocallbackg, which is where we'll find +// pointer-declared arguments. +// +// When fn is nil (frame is saved g), call dropm instead, +// this is used when the C thread is exiting. +func cgocallback(fn, frame, ctxt uintptr) + +func gogo(buf *gobuf) + +func asminit() +func setg(gg *g) +func breakpoint() + +// reflectcall calls fn with arguments described by stackArgs, stackArgsSize, +// frameSize, and regArgs. +// +// Arguments passed on the stack and space for return values passed on the stack +// must be laid out at the space pointed to by stackArgs (with total length +// stackArgsSize) according to the ABI. +// +// stackRetOffset must be some value <= stackArgsSize that indicates the +// offset within stackArgs where the return value space begins. +// +// frameSize is the total size of the argument frame at stackArgs and must +// therefore be >= stackArgsSize. It must include additional space for spilling +// register arguments for stack growth and preemption. +// +// TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, +// since frameSize will be redundant with stackArgsSize. +// +// Arguments passed in registers must be laid out in regArgs according to the ABI. +// regArgs will hold any return values passed in registers after the call. +// +// reflectcall copies stack arguments from stackArgs to the goroutine stack, and +// then copies back stackArgsSize-stackRetOffset bytes back to the return space +// in stackArgs once fn has completed. It also "unspills" argument registers from +// regArgs before calling fn, and spills them back into regArgs immediately +// following the call to fn. If there are results being returned on the stack, +// the caller should pass the argument frame type as stackArgsType so that +// reflectcall can execute appropriate write barriers during the copy. +// +// reflectcall expects regArgs.ReturnIsPtr to be populated indicating which +// registers on the return path will contain Go pointers. It will then store +// these pointers in regArgs.Ptrs such that they are visible to the GC. +// +// Package reflect passes a frame type. In package runtime, there is only +// one call that copies results back, in callbackWrap in syscall_windows.go, and it +// does NOT pass a frame type, meaning there are no write barriers invoked. See that +// call site for justification. +// +// Package reflect accesses this symbol through a linkname. +// +// Arguments passed through to reflectcall do not escape. The type is used +// only in a very limited callee of reflectcall, the stackArgs are copied, and +// regArgs is only used in the reflectcall frame. +// +//go:noescape +func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) + +func procyield(cycles uint32) + +type neverCallThisFunction struct{} + +// goexit is the return stub at the top of every goroutine call stack. +// Each goroutine stack is constructed as if goexit called the +// goroutine's entry point function, so that when the entry point +// function returns, it will return to goexit, which will call goexit1 +// to perform the actual exit. +// +// This function must never be called directly. Call goexit1 instead. +// gentraceback assumes that goexit terminates the stack. A direct +// call on the stack will cause gentraceback to stop walking the stack +// prematurely and if there is leftover state it may panic. +func goexit(neverCallThisFunction) + +// publicationBarrier performs a store/store barrier (a "publication" +// or "export" barrier). Some form of synchronization is required +// between initializing an object and making that object accessible to +// another processor. Without synchronization, the initialization +// writes and the "publication" write may be reordered, allowing the +// other processor to follow the pointer and observe an uninitialized +// object. In general, higher-level synchronization should be used, +// such as locking or an atomic pointer write. publicationBarrier is +// for when those aren't an option, such as in the implementation of +// the memory manager. +// +// There's no corresponding barrier for the read side because the read +// side naturally has a data dependency order. All architectures that +// Go supports or seems likely to ever support automatically enforce +// data dependency ordering. +func publicationBarrier() + +// getcallerpc returns the program counter (PC) of its caller's caller. +// getcallersp returns the stack pointer (SP) of its caller's caller. +// The implementation may be a compiler intrinsic; there is not +// necessarily code implementing this on every platform. +// +// For example: +// +// func f(arg1, arg2, arg3 int) { +// pc := getcallerpc() +// sp := getcallersp() +// } +// +// These two lines find the PC and SP immediately following +// the call to f (where f will return). +// +// The call to getcallerpc and getcallersp must be done in the +// frame being asked about. +// +// The result of getcallersp is correct at the time of the return, +// but it may be invalidated by any subsequent call to a function +// that might relocate the stack in order to grow or shrink it. +// A general rule is that the result of getcallersp should be used +// immediately and can only be passed to nosplit functions. + +//go:noescape +func getcallerpc() uintptr + +//go:noescape +func getcallersp() uintptr // implemented as an intrinsic on all platforms + +// getclosureptr returns the pointer to the current closure. +// getclosureptr can only be used in an assignment statement +// at the entry of a function. Moreover, go:nosplit directive +// must be specified at the declaration of caller function, +// so that the function prolog does not clobber the closure register. +// for example: +// +// //go:nosplit +// func f(arg1, arg2, arg3 int) { +// dx := getclosureptr() +// } +// +// The compiler rewrites calls to this function into instructions that fetch the +// pointer from a well-known register (DX on x86 architecture, etc.) directly. +func getclosureptr() uintptr + +//go:noescape +func asmcgocall(fn, arg unsafe.Pointer) int32 + +func morestack() +func morestack_noctxt() +func rt0_go() + +// return0 is a stub used to return 0 from deferproc. +// It is called at the very end of deferproc to signal +// the calling Go function that it should not jump +// to deferreturn. +// in asm_*.s +func return0() + +// in asm_*.s +// not called directly; definitions here supply type information for traceback. +// These must have the same signature (arg pointer map) as reflectcall. +func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) +func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) + +func systemstack_switch() + +// alignUp rounds n up to a multiple of a. a must be a power of 2. +func alignUp(n, a uintptr) uintptr { + return (n + a - 1) &^ (a - 1) +} + +// alignDown rounds n down to a multiple of a. a must be a power of 2. +func alignDown(n, a uintptr) uintptr { + return n &^ (a - 1) +} + +// divRoundUp returns ceil(n / a). +func divRoundUp(n, a uintptr) uintptr { + // a is generally a power of two. This will get inlined and + // the compiler will optimize the division. + return (n + a - 1) / a +} + +// checkASM reports whether assembly runtime checks have passed. +func checkASM() bool + +func memequal_varlen(a, b unsafe.Pointer) bool + +// bool2int returns 0 if x is false or 1 if x is true. +func bool2int(x bool) int { + // Avoid branches. In the SSA compiler, this compiles to + // exactly what you would want it to. + return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) +} + +// abort crashes the runtime in situations where even throw might not +// work. In general it should do something a debugger will recognize +// (e.g., an INT3 on x86). A crash in abort is recognized by the +// signal handler, which will attempt to tear down the runtime +// immediately. +func abort() + +// Called from compiled code; declared for vet; do NOT call from Go. +func gcWriteBarrier1() +func gcWriteBarrier2() +func gcWriteBarrier3() +func gcWriteBarrier4() +func gcWriteBarrier5() +func gcWriteBarrier6() +func gcWriteBarrier7() +func gcWriteBarrier8() +func duffzero() +func duffcopy() + +// Called from linker-generated .initarray; declared for go vet; do NOT call from Go. +func addmoduledata() + +// Injected by the signal handler for panicking signals. +// Initializes any registers that have fixed meaning at calls but +// are scratch in bodies and calls sigpanic. +// On many platforms it just jumps to sigpanic. +func sigpanic0() + +// intArgRegs is used by the various register assignment +// algorithm implementations in the runtime. These include:. +// - Finalizers (mfinal.go) +// - Windows callbacks (syscall_windows.go) +// +// Both are stripped-down versions of the algorithm since they +// only have to deal with a subset of cases (finalizers only +// take a pointer or interface argument, Go Windows callbacks +// don't support floating point). +// +// It should be modified with care and are generally only +// modified when testing this package. +// +// It should never be set higher than its internal/abi +// constant counterparts, because the system relies on a +// structure that is at least large enough to hold the +// registers the system supports. +// +// Protected by finlock. +var intArgRegs = abi.IntArgRegs |