summaryrefslogtreecommitdiffstats
path: root/src/runtime/panic.go
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-16 19:25:22 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-16 19:25:22 +0000
commitf6ad4dcef54c5ce997a4bad5a6d86de229015700 (patch)
tree7cfa4e31ace5c2bd95c72b154d15af494b2bcbef /src/runtime/panic.go
parentInitial commit. (diff)
downloadgolang-1.22-f6ad4dcef54c5ce997a4bad5a6d86de229015700.tar.xz
golang-1.22-f6ad4dcef54c5ce997a4bad5a6d86de229015700.zip
Adding upstream version 1.22.1.upstream/1.22.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--src/runtime/panic.go1448
1 files changed, 1448 insertions, 0 deletions
diff --git a/src/runtime/panic.go b/src/runtime/panic.go
new file mode 100644
index 0000000..36d658a
--- /dev/null
+++ b/src/runtime/panic.go
@@ -0,0 +1,1448 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+ "internal/abi"
+ "internal/goarch"
+ "runtime/internal/atomic"
+ "runtime/internal/sys"
+ "unsafe"
+)
+
+// throwType indicates the current type of ongoing throw, which affects the
+// amount of detail printed to stderr. Higher values include more detail.
+type throwType uint32
+
+const (
+ // throwTypeNone means that we are not throwing.
+ throwTypeNone throwType = iota
+
+ // throwTypeUser is a throw due to a problem with the application.
+ //
+ // These throws do not include runtime frames, system goroutines, or
+ // frame metadata.
+ throwTypeUser
+
+ // throwTypeRuntime is a throw due to a problem with Go itself.
+ //
+ // These throws include as much information as possible to aid in
+ // debugging the runtime, including runtime frames, system goroutines,
+ // and frame metadata.
+ throwTypeRuntime
+)
+
+// We have two different ways of doing defers. The older way involves creating a
+// defer record at the time that a defer statement is executing and adding it to a
+// defer chain. This chain is inspected by the deferreturn call at all function
+// exits in order to run the appropriate defer calls. A cheaper way (which we call
+// open-coded defers) is used for functions in which no defer statements occur in
+// loops. In that case, we simply store the defer function/arg information into
+// specific stack slots at the point of each defer statement, as well as setting a
+// bit in a bitmask. At each function exit, we add inline code to directly make
+// the appropriate defer calls based on the bitmask and fn/arg information stored
+// on the stack. During panic/Goexit processing, the appropriate defer calls are
+// made using extra funcdata info that indicates the exact stack slots that
+// contain the bitmask and defer fn/args.
+
+// Check to make sure we can really generate a panic. If the panic
+// was generated from the runtime, or from inside malloc, then convert
+// to a throw of msg.
+// pc should be the program counter of the compiler-generated code that
+// triggered this panic.
+func panicCheck1(pc uintptr, msg string) {
+ if goarch.IsWasm == 0 && hasPrefix(funcname(findfunc(pc)), "runtime.") {
+ // Note: wasm can't tail call, so we can't get the original caller's pc.
+ throw(msg)
+ }
+ // TODO: is this redundant? How could we be in malloc
+ // but not in the runtime? runtime/internal/*, maybe?
+ gp := getg()
+ if gp != nil && gp.m != nil && gp.m.mallocing != 0 {
+ throw(msg)
+ }
+}
+
+// Same as above, but calling from the runtime is allowed.
+//
+// Using this function is necessary for any panic that may be
+// generated by runtime.sigpanic, since those are always called by the
+// runtime.
+func panicCheck2(err string) {
+ // panic allocates, so to avoid recursive malloc, turn panics
+ // during malloc into throws.
+ gp := getg()
+ if gp != nil && gp.m != nil && gp.m.mallocing != 0 {
+ throw(err)
+ }
+}
+
+// Many of the following panic entry-points turn into throws when they
+// happen in various runtime contexts. These should never happen in
+// the runtime, and if they do, they indicate a serious issue and
+// should not be caught by user code.
+//
+// The panic{Index,Slice,divide,shift} functions are called by
+// code generated by the compiler for out of bounds index expressions,
+// out of bounds slice expressions, division by zero, and shift by negative.
+// The panicdivide (again), panicoverflow, panicfloat, and panicmem
+// functions are called by the signal handler when a signal occurs
+// indicating the respective problem.
+//
+// Since panic{Index,Slice,shift} are never called directly, and
+// since the runtime package should never have an out of bounds slice
+// or array reference or negative shift, if we see those functions called from the
+// runtime package we turn the panic into a throw. That will dump the
+// entire runtime stack for easier debugging.
+//
+// The entry points called by the signal handler will be called from
+// runtime.sigpanic, so we can't disallow calls from the runtime to
+// these (they always look like they're called from the runtime).
+// Hence, for these, we just check for clearly bad runtime conditions.
+//
+// The panic{Index,Slice} functions are implemented in assembly and tail call
+// to the goPanic{Index,Slice} functions below. This is done so we can use
+// a space-minimal register calling convention.
+
+// failures in the comparisons for s[x], 0 <= x < y (y == len(s))
+//
+//go:yeswritebarrierrec
+func goPanicIndex(x int, y int) {
+ panicCheck1(getcallerpc(), "index out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsIndex})
+}
+
+//go:yeswritebarrierrec
+func goPanicIndexU(x uint, y int) {
+ panicCheck1(getcallerpc(), "index out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsIndex})
+}
+
+// failures in the comparisons for s[:x], 0 <= x <= y (y == len(s) or cap(s))
+//
+//go:yeswritebarrierrec
+func goPanicSliceAlen(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAlen})
+}
+
+//go:yeswritebarrierrec
+func goPanicSliceAlenU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAlen})
+}
+
+//go:yeswritebarrierrec
+func goPanicSliceAcap(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceAcap})
+}
+
+//go:yeswritebarrierrec
+func goPanicSliceAcapU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceAcap})
+}
+
+// failures in the comparisons for s[x:y], 0 <= x <= y
+//
+//go:yeswritebarrierrec
+func goPanicSliceB(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSliceB})
+}
+
+//go:yeswritebarrierrec
+func goPanicSliceBU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSliceB})
+}
+
+// failures in the comparisons for s[::x], 0 <= x <= y (y == len(s) or cap(s))
+func goPanicSlice3Alen(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Alen})
+}
+func goPanicSlice3AlenU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Alen})
+}
+func goPanicSlice3Acap(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3Acap})
+}
+func goPanicSlice3AcapU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3Acap})
+}
+
+// failures in the comparisons for s[:x:y], 0 <= x <= y
+func goPanicSlice3B(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3B})
+}
+func goPanicSlice3BU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3B})
+}
+
+// failures in the comparisons for s[x:y:], 0 <= x <= y
+func goPanicSlice3C(x int, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsSlice3C})
+}
+func goPanicSlice3CU(x uint, y int) {
+ panicCheck1(getcallerpc(), "slice bounds out of range")
+ panic(boundsError{x: int64(x), signed: false, y: y, code: boundsSlice3C})
+}
+
+// failures in the conversion ([x]T)(s) or (*[x]T)(s), 0 <= x <= y, y == len(s)
+func goPanicSliceConvert(x int, y int) {
+ panicCheck1(getcallerpc(), "slice length too short to convert to array or pointer to array")
+ panic(boundsError{x: int64(x), signed: true, y: y, code: boundsConvert})
+}
+
+// Implemented in assembly, as they take arguments in registers.
+// Declared here to mark them as ABIInternal.
+func panicIndex(x int, y int)
+func panicIndexU(x uint, y int)
+func panicSliceAlen(x int, y int)
+func panicSliceAlenU(x uint, y int)
+func panicSliceAcap(x int, y int)
+func panicSliceAcapU(x uint, y int)
+func panicSliceB(x int, y int)
+func panicSliceBU(x uint, y int)
+func panicSlice3Alen(x int, y int)
+func panicSlice3AlenU(x uint, y int)
+func panicSlice3Acap(x int, y int)
+func panicSlice3AcapU(x uint, y int)
+func panicSlice3B(x int, y int)
+func panicSlice3BU(x uint, y int)
+func panicSlice3C(x int, y int)
+func panicSlice3CU(x uint, y int)
+func panicSliceConvert(x int, y int)
+
+var shiftError = error(errorString("negative shift amount"))
+
+//go:yeswritebarrierrec
+func panicshift() {
+ panicCheck1(getcallerpc(), "negative shift amount")
+ panic(shiftError)
+}
+
+var divideError = error(errorString("integer divide by zero"))
+
+//go:yeswritebarrierrec
+func panicdivide() {
+ panicCheck2("integer divide by zero")
+ panic(divideError)
+}
+
+var overflowError = error(errorString("integer overflow"))
+
+func panicoverflow() {
+ panicCheck2("integer overflow")
+ panic(overflowError)
+}
+
+var floatError = error(errorString("floating point error"))
+
+func panicfloat() {
+ panicCheck2("floating point error")
+ panic(floatError)
+}
+
+var memoryError = error(errorString("invalid memory address or nil pointer dereference"))
+
+func panicmem() {
+ panicCheck2("invalid memory address or nil pointer dereference")
+ panic(memoryError)
+}
+
+func panicmemAddr(addr uintptr) {
+ panicCheck2("invalid memory address or nil pointer dereference")
+ panic(errorAddressString{msg: "invalid memory address or nil pointer dereference", addr: addr})
+}
+
+// Create a new deferred function fn, which has no arguments and results.
+// The compiler turns a defer statement into a call to this.
+func deferproc(fn func()) {
+ gp := getg()
+ if gp.m.curg != gp {
+ // go code on the system stack can't defer
+ throw("defer on system stack")
+ }
+
+ d := newdefer()
+ d.link = gp._defer
+ gp._defer = d
+ d.fn = fn
+ d.pc = getcallerpc()
+ // We must not be preempted between calling getcallersp and
+ // storing it to d.sp because getcallersp's result is a
+ // uintptr stack pointer.
+ d.sp = getcallersp()
+
+ // deferproc returns 0 normally.
+ // a deferred func that stops a panic
+ // makes the deferproc return 1.
+ // the code the compiler generates always
+ // checks the return value and jumps to the
+ // end of the function if deferproc returns != 0.
+ return0()
+ // No code can go here - the C return register has
+ // been set and must not be clobbered.
+}
+
+var rangeExitError = error(errorString("range function continued iteration after exit"))
+
+//go:noinline
+func panicrangeexit() {
+ panic(rangeExitError)
+}
+
+// deferrangefunc is called by functions that are about to
+// execute a range-over-function loop in which the loop body
+// may execute a defer statement. That defer needs to add to
+// the chain for the current function, not the func literal synthesized
+// to represent the loop body. To do that, the original function
+// calls deferrangefunc to obtain an opaque token representing
+// the current frame, and then the loop body uses deferprocat
+// instead of deferproc to add to that frame's defer lists.
+//
+// The token is an 'any' with underlying type *atomic.Pointer[_defer].
+// It is the atomically-updated head of a linked list of _defer structs
+// representing deferred calls. At the same time, we create a _defer
+// struct on the main g._defer list with d.head set to this head pointer.
+//
+// The g._defer list is now a linked list of deferred calls,
+// but an atomic list hanging off:
+//
+// g._defer => d4 -> d3 -> drangefunc -> d2 -> d1 -> nil
+// | .head
+// |
+// +--> dY -> dX -> nil
+//
+// with each -> indicating a d.link pointer, and where drangefunc
+// has the d.rangefunc = true bit set.
+// Note that the function being ranged over may have added
+// its own defers (d4 and d3), so drangefunc need not be at the
+// top of the list when deferprocat is used. This is why we pass
+// the atomic head explicitly.
+//
+// To keep misbehaving programs from crashing the runtime,
+// deferprocat pushes new defers onto the .head list atomically.
+// The fact that it is a separate list from the main goroutine
+// defer list means that the main goroutine's defers can still
+// be handled non-atomically.
+//
+// In the diagram, dY and dX are meant to be processed when
+// drangefunc would be processed, which is to say the defer order
+// should be d4, d3, dY, dX, d2, d1. To make that happen,
+// when defer processing reaches a d with rangefunc=true,
+// it calls deferconvert to atomically take the extras
+// away from d.head and then adds them to the main list.
+//
+// That is, deferconvert changes this list:
+//
+// g._defer => drangefunc -> d2 -> d1 -> nil
+// | .head
+// |
+// +--> dY -> dX -> nil
+//
+// into this list:
+//
+// g._defer => dY -> dX -> d2 -> d1 -> nil
+//
+// It also poisons *drangefunc.head so that any future
+// deferprocat using that head will throw.
+// (The atomic head is ordinary garbage collected memory so that
+// it's not a problem if user code holds onto it beyond
+// the lifetime of drangefunc.)
+//
+// TODO: We could arrange for the compiler to call into the
+// runtime after the loop finishes normally, to do an eager
+// deferconvert, which would catch calling the loop body
+// and having it defer after the loop is done. If we have a
+// more general catch of loop body misuse, though, this
+// might not be worth worrying about in addition.
+//
+// See also ../cmd/compile/internal/rangefunc/rewrite.go.
+func deferrangefunc() any {
+ gp := getg()
+ if gp.m.curg != gp {
+ // go code on the system stack can't defer
+ throw("defer on system stack")
+ }
+
+ d := newdefer()
+ d.link = gp._defer
+ gp._defer = d
+ d.pc = getcallerpc()
+ // We must not be preempted between calling getcallersp and
+ // storing it to d.sp because getcallersp's result is a
+ // uintptr stack pointer.
+ d.sp = getcallersp()
+
+ d.rangefunc = true
+ d.head = new(atomic.Pointer[_defer])
+
+ return d.head
+}
+
+// badDefer returns a fixed bad defer pointer for poisoning an atomic defer list head.
+func badDefer() *_defer {
+ return (*_defer)(unsafe.Pointer(uintptr(1)))
+}
+
+// deferprocat is like deferproc but adds to the atomic list represented by frame.
+// See the doc comment for deferrangefunc for details.
+func deferprocat(fn func(), frame any) {
+ head := frame.(*atomic.Pointer[_defer])
+ if raceenabled {
+ racewritepc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferprocat))
+ }
+ d1 := newdefer()
+ d1.fn = fn
+ for {
+ d1.link = head.Load()
+ if d1.link == badDefer() {
+ throw("defer after range func returned")
+ }
+ if head.CompareAndSwap(d1.link, d1) {
+ break
+ }
+ }
+
+ // Must be last - see deferproc above.
+ return0()
+}
+
+// deferconvert converts a rangefunc defer list into an ordinary list.
+// See the doc comment for deferrangefunc for details.
+func deferconvert(d *_defer) *_defer {
+ head := d.head
+ if raceenabled {
+ racereadpc(unsafe.Pointer(head), getcallerpc(), abi.FuncPCABIInternal(deferconvert))
+ }
+ tail := d.link
+ d.rangefunc = false
+ d0 := d
+
+ for {
+ d = head.Load()
+ if head.CompareAndSwap(d, badDefer()) {
+ break
+ }
+ }
+ if d == nil {
+ freedefer(d0)
+ return tail
+ }
+ for d1 := d; ; d1 = d1.link {
+ d1.sp = d0.sp
+ d1.pc = d0.pc
+ if d1.link == nil {
+ d1.link = tail
+ break
+ }
+ }
+ freedefer(d0)
+ return d
+}
+
+// deferprocStack queues a new deferred function with a defer record on the stack.
+// The defer record must have its fn field initialized.
+// All other fields can contain junk.
+// Nosplit because of the uninitialized pointer fields on the stack.
+//
+//go:nosplit
+func deferprocStack(d *_defer) {
+ gp := getg()
+ if gp.m.curg != gp {
+ // go code on the system stack can't defer
+ throw("defer on system stack")
+ }
+ // fn is already set.
+ // The other fields are junk on entry to deferprocStack and
+ // are initialized here.
+ d.heap = false
+ d.rangefunc = false
+ d.sp = getcallersp()
+ d.pc = getcallerpc()
+ // The lines below implement:
+ // d.panic = nil
+ // d.fd = nil
+ // d.link = gp._defer
+ // d.head = nil
+ // gp._defer = d
+ // But without write barriers. The first three are writes to
+ // the stack so they don't need a write barrier, and furthermore
+ // are to uninitialized memory, so they must not use a write barrier.
+ // The fourth write does not require a write barrier because we
+ // explicitly mark all the defer structures, so we don't need to
+ // keep track of pointers to them with a write barrier.
+ *(*uintptr)(unsafe.Pointer(&d.link)) = uintptr(unsafe.Pointer(gp._defer))
+ *(*uintptr)(unsafe.Pointer(&d.head)) = 0
+ *(*uintptr)(unsafe.Pointer(&gp._defer)) = uintptr(unsafe.Pointer(d))
+
+ return0()
+ // No code can go here - the C return register has
+ // been set and must not be clobbered.
+}
+
+// Each P holds a pool for defers.
+
+// Allocate a Defer, usually using per-P pool.
+// Each defer must be released with freedefer. The defer is not
+// added to any defer chain yet.
+func newdefer() *_defer {
+ var d *_defer
+ mp := acquirem()
+ pp := mp.p.ptr()
+ if len(pp.deferpool) == 0 && sched.deferpool != nil {
+ lock(&sched.deferlock)
+ for len(pp.deferpool) < cap(pp.deferpool)/2 && sched.deferpool != nil {
+ d := sched.deferpool
+ sched.deferpool = d.link
+ d.link = nil
+ pp.deferpool = append(pp.deferpool, d)
+ }
+ unlock(&sched.deferlock)
+ }
+ if n := len(pp.deferpool); n > 0 {
+ d = pp.deferpool[n-1]
+ pp.deferpool[n-1] = nil
+ pp.deferpool = pp.deferpool[:n-1]
+ }
+ releasem(mp)
+ mp, pp = nil, nil
+
+ if d == nil {
+ // Allocate new defer.
+ d = new(_defer)
+ }
+ d.heap = true
+ return d
+}
+
+// Free the given defer.
+// The defer cannot be used after this call.
+//
+// This is nosplit because the incoming defer is in a perilous state.
+// It's not on any defer list, so stack copying won't adjust stack
+// pointers in it (namely, d.link). Hence, if we were to copy the
+// stack, d could then contain a stale pointer.
+//
+//go:nosplit
+func freedefer(d *_defer) {
+ d.link = nil
+ // After this point we can copy the stack.
+
+ if d.fn != nil {
+ freedeferfn()
+ }
+ if !d.heap {
+ return
+ }
+
+ mp := acquirem()
+ pp := mp.p.ptr()
+ if len(pp.deferpool) == cap(pp.deferpool) {
+ // Transfer half of local cache to the central cache.
+ var first, last *_defer
+ for len(pp.deferpool) > cap(pp.deferpool)/2 {
+ n := len(pp.deferpool)
+ d := pp.deferpool[n-1]
+ pp.deferpool[n-1] = nil
+ pp.deferpool = pp.deferpool[:n-1]
+ if first == nil {
+ first = d
+ } else {
+ last.link = d
+ }
+ last = d
+ }
+ lock(&sched.deferlock)
+ last.link = sched.deferpool
+ sched.deferpool = first
+ unlock(&sched.deferlock)
+ }
+
+ *d = _defer{}
+
+ pp.deferpool = append(pp.deferpool, d)
+
+ releasem(mp)
+ mp, pp = nil, nil
+}
+
+// Separate function so that it can split stack.
+// Windows otherwise runs out of stack space.
+func freedeferfn() {
+ // fn must be cleared before d is unlinked from gp.
+ throw("freedefer with d.fn != nil")
+}
+
+// deferreturn runs deferred functions for the caller's frame.
+// The compiler inserts a call to this at the end of any
+// function which calls defer.
+func deferreturn() {
+ var p _panic
+ p.deferreturn = true
+
+ p.start(getcallerpc(), unsafe.Pointer(getcallersp()))
+ for {
+ fn, ok := p.nextDefer()
+ if !ok {
+ break
+ }
+ fn()
+ }
+}
+
+// Goexit terminates the goroutine that calls it. No other goroutine is affected.
+// Goexit runs all deferred calls before terminating the goroutine. Because Goexit
+// is not a panic, any recover calls in those deferred functions will return nil.
+//
+// Calling Goexit from the main goroutine terminates that goroutine
+// without func main returning. Since func main has not returned,
+// the program continues execution of other goroutines.
+// If all other goroutines exit, the program crashes.
+func Goexit() {
+ // Create a panic object for Goexit, so we can recognize when it might be
+ // bypassed by a recover().
+ var p _panic
+ p.goexit = true
+
+ p.start(getcallerpc(), unsafe.Pointer(getcallersp()))
+ for {
+ fn, ok := p.nextDefer()
+ if !ok {
+ break
+ }
+ fn()
+ }
+
+ goexit1()
+}
+
+// Call all Error and String methods before freezing the world.
+// Used when crashing with panicking.
+func preprintpanics(p *_panic) {
+ defer func() {
+ text := "panic while printing panic value"
+ switch r := recover().(type) {
+ case nil:
+ // nothing to do
+ case string:
+ throw(text + ": " + r)
+ default:
+ throw(text + ": type " + toRType(efaceOf(&r)._type).string())
+ }
+ }()
+ for p != nil {
+ switch v := p.arg.(type) {
+ case error:
+ p.arg = v.Error()
+ case stringer:
+ p.arg = v.String()
+ }
+ p = p.link
+ }
+}
+
+// Print all currently active panics. Used when crashing.
+// Should only be called after preprintpanics.
+func printpanics(p *_panic) {
+ if p.link != nil {
+ printpanics(p.link)
+ if !p.link.goexit {
+ print("\t")
+ }
+ }
+ if p.goexit {
+ return
+ }
+ print("panic: ")
+ printany(p.arg)
+ if p.recovered {
+ print(" [recovered]")
+ }
+ print("\n")
+}
+
+// readvarintUnsafe reads the uint32 in varint format starting at fd, and returns the
+// uint32 and a pointer to the byte following the varint.
+//
+// The implementation is the same with runtime.readvarint, except that this function
+// uses unsafe.Pointer for speed.
+func readvarintUnsafe(fd unsafe.Pointer) (uint32, unsafe.Pointer) {
+ var r uint32
+ var shift int
+ for {
+ b := *(*uint8)(fd)
+ fd = add(fd, unsafe.Sizeof(b))
+ if b < 128 {
+ return r + uint32(b)<<shift, fd
+ }
+ r += uint32(b&0x7F) << (shift & 31)
+ shift += 7
+ if shift > 28 {
+ panic("Bad varint")
+ }
+ }
+}
+
+// A PanicNilError happens when code calls panic(nil).
+//
+// Before Go 1.21, programs that called panic(nil) observed recover returning nil.
+// Starting in Go 1.21, programs that call panic(nil) observe recover returning a *PanicNilError.
+// Programs can change back to the old behavior by setting GODEBUG=panicnil=1.
+type PanicNilError struct {
+ // This field makes PanicNilError structurally different from
+ // any other struct in this package, and the _ makes it different
+ // from any struct in other packages too.
+ // This avoids any accidental conversions being possible
+ // between this struct and some other struct sharing the same fields,
+ // like happened in go.dev/issue/56603.
+ _ [0]*PanicNilError
+}
+
+func (*PanicNilError) Error() string { return "panic called with nil argument" }
+func (*PanicNilError) RuntimeError() {}
+
+var panicnil = &godebugInc{name: "panicnil"}
+
+// The implementation of the predeclared function panic.
+func gopanic(e any) {
+ if e == nil {
+ if debug.panicnil.Load() != 1 {
+ e = new(PanicNilError)
+ } else {
+ panicnil.IncNonDefault()
+ }
+ }
+
+ gp := getg()
+ if gp.m.curg != gp {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ throw("panic on system stack")
+ }
+
+ if gp.m.mallocing != 0 {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ throw("panic during malloc")
+ }
+ if gp.m.preemptoff != "" {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ print("preempt off reason: ")
+ print(gp.m.preemptoff)
+ print("\n")
+ throw("panic during preemptoff")
+ }
+ if gp.m.locks != 0 {
+ print("panic: ")
+ printany(e)
+ print("\n")
+ throw("panic holding locks")
+ }
+
+ var p _panic
+ p.arg = e
+
+ runningPanicDefers.Add(1)
+
+ p.start(getcallerpc(), unsafe.Pointer(getcallersp()))
+ for {
+ fn, ok := p.nextDefer()
+ if !ok {
+ break
+ }
+ fn()
+ }
+
+ // ran out of deferred calls - old-school panic now
+ // Because it is unsafe to call arbitrary user code after freezing
+ // the world, we call preprintpanics to invoke all necessary Error
+ // and String methods to prepare the panic strings before startpanic.
+ preprintpanics(&p)
+
+ fatalpanic(&p) // should not return
+ *(*int)(nil) = 0 // not reached
+}
+
+// start initializes a panic to start unwinding the stack.
+//
+// If p.goexit is true, then start may return multiple times.
+func (p *_panic) start(pc uintptr, sp unsafe.Pointer) {
+ gp := getg()
+
+ // Record the caller's PC and SP, so recovery can identify panics
+ // that have been recovered. Also, so that if p is from Goexit, we
+ // can restart its defer processing loop if a recovered panic tries
+ // to jump past it.
+ p.startPC = getcallerpc()
+ p.startSP = unsafe.Pointer(getcallersp())
+
+ if p.deferreturn {
+ p.sp = sp
+
+ if s := (*savedOpenDeferState)(gp.param); s != nil {
+ // recovery saved some state for us, so that we can resume
+ // calling open-coded defers without unwinding the stack.
+
+ gp.param = nil
+
+ p.retpc = s.retpc
+ p.deferBitsPtr = (*byte)(add(sp, s.deferBitsOffset))
+ p.slotsPtr = add(sp, s.slotsOffset)
+ }
+
+ return
+ }
+
+ p.link = gp._panic
+ gp._panic = (*_panic)(noescape(unsafe.Pointer(p)))
+
+ // Initialize state machine, and find the first frame with a defer.
+ //
+ // Note: We could use startPC and startSP here, but callers will
+ // never have defer statements themselves. By starting at their
+ // caller instead, we avoid needing to unwind through an extra
+ // frame. It also somewhat simplifies the terminating condition for
+ // deferreturn.
+ p.lr, p.fp = pc, sp
+ p.nextFrame()
+}
+
+// nextDefer returns the next deferred function to invoke, if any.
+//
+// Note: The "ok bool" result is necessary to correctly handle when
+// the deferred function itself was nil (e.g., "defer (func())(nil)").
+func (p *_panic) nextDefer() (func(), bool) {
+ gp := getg()
+
+ if !p.deferreturn {
+ if gp._panic != p {
+ throw("bad panic stack")
+ }
+
+ if p.recovered {
+ mcall(recovery) // does not return
+ throw("recovery failed")
+ }
+ }
+
+ // The assembler adjusts p.argp in wrapper functions that shouldn't
+ // be visible to recover(), so we need to restore it each iteration.
+ p.argp = add(p.startSP, sys.MinFrameSize)
+
+ for {
+ for p.deferBitsPtr != nil {
+ bits := *p.deferBitsPtr
+
+ // Check whether any open-coded defers are still pending.
+ //
+ // Note: We need to check this upfront (rather than after
+ // clearing the top bit) because it's possible that Goexit
+ // invokes a deferred call, and there were still more pending
+ // open-coded defers in the frame; but then the deferred call
+ // panic and invoked the remaining defers in the frame, before
+ // recovering and restarting the Goexit loop.
+ if bits == 0 {
+ p.deferBitsPtr = nil
+ break
+ }
+
+ // Find index of top bit set.
+ i := 7 - uintptr(sys.LeadingZeros8(bits))
+
+ // Clear bit and store it back.
+ bits &^= 1 << i
+ *p.deferBitsPtr = bits
+
+ return *(*func())(add(p.slotsPtr, i*goarch.PtrSize)), true
+ }
+
+ Recheck:
+ if d := gp._defer; d != nil && d.sp == uintptr(p.sp) {
+ if d.rangefunc {
+ gp._defer = deferconvert(d)
+ goto Recheck
+ }
+
+ fn := d.fn
+ d.fn = nil
+
+ // TODO(mdempsky): Instead of having each deferproc call have
+ // its own "deferreturn(); return" sequence, we should just make
+ // them reuse the one we emit for open-coded defers.
+ p.retpc = d.pc
+
+ // Unlink and free.
+ gp._defer = d.link
+ freedefer(d)
+
+ return fn, true
+ }
+
+ if !p.nextFrame() {
+ return nil, false
+ }
+ }
+}
+
+// nextFrame finds the next frame that contains deferred calls, if any.
+func (p *_panic) nextFrame() (ok bool) {
+ if p.lr == 0 {
+ return false
+ }
+
+ gp := getg()
+ systemstack(func() {
+ var limit uintptr
+ if d := gp._defer; d != nil {
+ limit = d.sp
+ }
+
+ var u unwinder
+ u.initAt(p.lr, uintptr(p.fp), 0, gp, 0)
+ for {
+ if !u.valid() {
+ p.lr = 0
+ return // ok == false
+ }
+
+ // TODO(mdempsky): If we populate u.frame.fn.deferreturn for
+ // every frame containing a defer (not just open-coded defers),
+ // then we can simply loop until we find the next frame where
+ // it's non-zero.
+
+ if u.frame.sp == limit {
+ break // found a frame with linked defers
+ }
+
+ if p.initOpenCodedDefers(u.frame.fn, unsafe.Pointer(u.frame.varp)) {
+ break // found a frame with open-coded defers
+ }
+
+ u.next()
+ }
+
+ p.lr = u.frame.lr
+ p.sp = unsafe.Pointer(u.frame.sp)
+ p.fp = unsafe.Pointer(u.frame.fp)
+
+ ok = true
+ })
+
+ return
+}
+
+func (p *_panic) initOpenCodedDefers(fn funcInfo, varp unsafe.Pointer) bool {
+ fd := funcdata(fn, abi.FUNCDATA_OpenCodedDeferInfo)
+ if fd == nil {
+ return false
+ }
+
+ if fn.deferreturn == 0 {
+ throw("missing deferreturn")
+ }
+
+ deferBitsOffset, fd := readvarintUnsafe(fd)
+ deferBitsPtr := (*uint8)(add(varp, -uintptr(deferBitsOffset)))
+ if *deferBitsPtr == 0 {
+ return false // has open-coded defers, but none pending
+ }
+
+ slotsOffset, fd := readvarintUnsafe(fd)
+
+ p.retpc = fn.entry() + uintptr(fn.deferreturn)
+ p.deferBitsPtr = deferBitsPtr
+ p.slotsPtr = add(varp, -uintptr(slotsOffset))
+
+ return true
+}
+
+// The implementation of the predeclared function recover.
+// Cannot split the stack because it needs to reliably
+// find the stack segment of its caller.
+//
+// TODO(rsc): Once we commit to CopyStackAlways,
+// this doesn't need to be nosplit.
+//
+//go:nosplit
+func gorecover(argp uintptr) any {
+ // Must be in a function running as part of a deferred call during the panic.
+ // Must be called from the topmost function of the call
+ // (the function used in the defer statement).
+ // p.argp is the argument pointer of that topmost deferred function call.
+ // Compare against argp reported by caller.
+ // If they match, the caller is the one who can recover.
+ gp := getg()
+ p := gp._panic
+ if p != nil && !p.goexit && !p.recovered && argp == uintptr(p.argp) {
+ p.recovered = true
+ return p.arg
+ }
+ return nil
+}
+
+//go:linkname sync_throw sync.throw
+func sync_throw(s string) {
+ throw(s)
+}
+
+//go:linkname sync_fatal sync.fatal
+func sync_fatal(s string) {
+ fatal(s)
+}
+
+// throw triggers a fatal error that dumps a stack trace and exits.
+//
+// throw should be used for runtime-internal fatal errors where Go itself,
+// rather than user code, may be at fault for the failure.
+//
+//go:nosplit
+func throw(s string) {
+ // Everything throw does should be recursively nosplit so it
+ // can be called even when it's unsafe to grow the stack.
+ systemstack(func() {
+ print("fatal error: ", s, "\n")
+ })
+
+ fatalthrow(throwTypeRuntime)
+}
+
+// fatal triggers a fatal error that dumps a stack trace and exits.
+//
+// fatal is equivalent to throw, but is used when user code is expected to be
+// at fault for the failure, such as racing map writes.
+//
+// fatal does not include runtime frames, system goroutines, or frame metadata
+// (fp, sp, pc) in the stack trace unless GOTRACEBACK=system or higher.
+//
+//go:nosplit
+func fatal(s string) {
+ // Everything fatal does should be recursively nosplit so it
+ // can be called even when it's unsafe to grow the stack.
+ systemstack(func() {
+ print("fatal error: ", s, "\n")
+ })
+
+ fatalthrow(throwTypeUser)
+}
+
+// runningPanicDefers is non-zero while running deferred functions for panic.
+// This is used to try hard to get a panic stack trace out when exiting.
+var runningPanicDefers atomic.Uint32
+
+// panicking is non-zero when crashing the program for an unrecovered panic.
+var panicking atomic.Uint32
+
+// paniclk is held while printing the panic information and stack trace,
+// so that two concurrent panics don't overlap their output.
+var paniclk mutex
+
+// Unwind the stack after a deferred function calls recover
+// after a panic. Then arrange to continue running as though
+// the caller of the deferred function returned normally.
+//
+// However, if unwinding the stack would skip over a Goexit call, we
+// return into the Goexit loop instead, so it can continue processing
+// defers instead.
+func recovery(gp *g) {
+ p := gp._panic
+ pc, sp, fp := p.retpc, uintptr(p.sp), uintptr(p.fp)
+ p0, saveOpenDeferState := p, p.deferBitsPtr != nil && *p.deferBitsPtr != 0
+
+ // Unwind the panic stack.
+ for ; p != nil && uintptr(p.startSP) < sp; p = p.link {
+ // Don't allow jumping past a pending Goexit.
+ // Instead, have its _panic.start() call return again.
+ //
+ // TODO(mdempsky): In this case, Goexit will resume walking the
+ // stack where it left off, which means it will need to rewalk
+ // frames that we've already processed.
+ //
+ // There's a similar issue with nested panics, when the inner
+ // panic supercedes the outer panic. Again, we end up needing to
+ // walk the same stack frames.
+ //
+ // These are probably pretty rare occurrences in practice, and
+ // they don't seem any worse than the existing logic. But if we
+ // move the unwinding state into _panic, we could detect when we
+ // run into where the last panic started, and then just pick up
+ // where it left off instead.
+ //
+ // With how subtle defer handling is, this might not actually be
+ // worthwhile though.
+ if p.goexit {
+ pc, sp = p.startPC, uintptr(p.startSP)
+ saveOpenDeferState = false // goexit is unwinding the stack anyway
+ break
+ }
+
+ runningPanicDefers.Add(-1)
+ }
+ gp._panic = p
+
+ if p == nil { // must be done with signal
+ gp.sig = 0
+ }
+
+ if gp.param != nil {
+ throw("unexpected gp.param")
+ }
+ if saveOpenDeferState {
+ // If we're returning to deferreturn and there are more open-coded
+ // defers for it to call, save enough state for it to be able to
+ // pick up where p0 left off.
+ gp.param = unsafe.Pointer(&savedOpenDeferState{
+ retpc: p0.retpc,
+
+ // We need to save deferBitsPtr and slotsPtr too, but those are
+ // stack pointers. To avoid issues around heap objects pointing
+ // to the stack, save them as offsets from SP.
+ deferBitsOffset: uintptr(unsafe.Pointer(p0.deferBitsPtr)) - uintptr(p0.sp),
+ slotsOffset: uintptr(p0.slotsPtr) - uintptr(p0.sp),
+ })
+ }
+
+ // TODO(mdempsky): Currently, we rely on frames containing "defer"
+ // to end with "CALL deferreturn; RET". This allows deferreturn to
+ // finish running any pending defers in the frame.
+ //
+ // But we should be able to tell whether there are still pending
+ // defers here. If there aren't, we can just jump directly to the
+ // "RET" instruction. And if there are, we don't need an actual
+ // "CALL deferreturn" instruction; we can simulate it with something
+ // like:
+ //
+ // if usesLR {
+ // lr = pc
+ // } else {
+ // sp -= sizeof(pc)
+ // *(*uintptr)(sp) = pc
+ // }
+ // pc = funcPC(deferreturn)
+ //
+ // So that we effectively tail call into deferreturn, such that it
+ // then returns to the simple "RET" epilogue. That would save the
+ // overhead of the "deferreturn" call when there aren't actually any
+ // pending defers left, and shrink the TEXT size of compiled
+ // binaries. (Admittedly, both of these are modest savings.)
+
+ // Ensure we're recovering within the appropriate stack.
+ if sp != 0 && (sp < gp.stack.lo || gp.stack.hi < sp) {
+ print("recover: ", hex(sp), " not in [", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
+ throw("bad recovery")
+ }
+
+ // Make the deferproc for this d return again,
+ // this time returning 1. The calling function will
+ // jump to the standard return epilogue.
+ gp.sched.sp = sp
+ gp.sched.pc = pc
+ gp.sched.lr = 0
+ // Restore the bp on platforms that support frame pointers.
+ // N.B. It's fine to not set anything for platforms that don't
+ // support frame pointers, since nothing consumes them.
+ switch {
+ case goarch.IsAmd64 != 0:
+ // on x86, fp actually points one word higher than the top of
+ // the frame since the return address is saved on the stack by
+ // the caller
+ gp.sched.bp = fp - 2*goarch.PtrSize
+ case goarch.IsArm64 != 0:
+ // on arm64, the architectural bp points one word higher
+ // than the sp. fp is totally useless to us here, because it
+ // only gets us to the caller's fp.
+ gp.sched.bp = sp - goarch.PtrSize
+ }
+ gp.sched.ret = 1
+ gogo(&gp.sched)
+}
+
+// fatalthrow implements an unrecoverable runtime throw. It freezes the
+// system, prints stack traces starting from its caller, and terminates the
+// process.
+//
+//go:nosplit
+func fatalthrow(t throwType) {
+ pc := getcallerpc()
+ sp := getcallersp()
+ gp := getg()
+
+ if gp.m.throwing == throwTypeNone {
+ gp.m.throwing = t
+ }
+
+ // Switch to the system stack to avoid any stack growth, which may make
+ // things worse if the runtime is in a bad state.
+ systemstack(func() {
+ if isSecureMode() {
+ exit(2)
+ }
+
+ startpanic_m()
+
+ if dopanic_m(gp, pc, sp) {
+ // crash uses a decent amount of nosplit stack and we're already
+ // low on stack in throw, so crash on the system stack (unlike
+ // fatalpanic).
+ crash()
+ }
+
+ exit(2)
+ })
+
+ *(*int)(nil) = 0 // not reached
+}
+
+// fatalpanic implements an unrecoverable panic. It is like fatalthrow, except
+// that if msgs != nil, fatalpanic also prints panic messages and decrements
+// runningPanicDefers once main is blocked from exiting.
+//
+//go:nosplit
+func fatalpanic(msgs *_panic) {
+ pc := getcallerpc()
+ sp := getcallersp()
+ gp := getg()
+ var docrash bool
+ // Switch to the system stack to avoid any stack growth, which
+ // may make things worse if the runtime is in a bad state.
+ systemstack(func() {
+ if startpanic_m() && msgs != nil {
+ // There were panic messages and startpanic_m
+ // says it's okay to try to print them.
+
+ // startpanic_m set panicking, which will
+ // block main from exiting, so now OK to
+ // decrement runningPanicDefers.
+ runningPanicDefers.Add(-1)
+
+ printpanics(msgs)
+ }
+
+ docrash = dopanic_m(gp, pc, sp)
+ })
+
+ if docrash {
+ // By crashing outside the above systemstack call, debuggers
+ // will not be confused when generating a backtrace.
+ // Function crash is marked nosplit to avoid stack growth.
+ crash()
+ }
+
+ systemstack(func() {
+ exit(2)
+ })
+
+ *(*int)(nil) = 0 // not reached
+}
+
+// startpanic_m prepares for an unrecoverable panic.
+//
+// It returns true if panic messages should be printed, or false if
+// the runtime is in bad shape and should just print stacks.
+//
+// It must not have write barriers even though the write barrier
+// explicitly ignores writes once dying > 0. Write barriers still
+// assume that g.m.p != nil, and this function may not have P
+// in some contexts (e.g. a panic in a signal handler for a signal
+// sent to an M with no P).
+//
+//go:nowritebarrierrec
+func startpanic_m() bool {
+ gp := getg()
+ if mheap_.cachealloc.size == 0 { // very early
+ print("runtime: panic before malloc heap initialized\n")
+ }
+ // Disallow malloc during an unrecoverable panic. A panic
+ // could happen in a signal handler, or in a throw, or inside
+ // malloc itself. We want to catch if an allocation ever does
+ // happen (even if we're not in one of these situations).
+ gp.m.mallocing++
+
+ // If we're dying because of a bad lock count, set it to a
+ // good lock count so we don't recursively panic below.
+ if gp.m.locks < 0 {
+ gp.m.locks = 1
+ }
+
+ switch gp.m.dying {
+ case 0:
+ // Setting dying >0 has the side-effect of disabling this G's writebuf.
+ gp.m.dying = 1
+ panicking.Add(1)
+ lock(&paniclk)
+ if debug.schedtrace > 0 || debug.scheddetail > 0 {
+ schedtrace(true)
+ }
+ freezetheworld()
+ return true
+ case 1:
+ // Something failed while panicking.
+ // Just print a stack trace and exit.
+ gp.m.dying = 2
+ print("panic during panic\n")
+ return false
+ case 2:
+ // This is a genuine bug in the runtime, we couldn't even
+ // print the stack trace successfully.
+ gp.m.dying = 3
+ print("stack trace unavailable\n")
+ exit(4)
+ fallthrough
+ default:
+ // Can't even print! Just exit.
+ exit(5)
+ return false // Need to return something.
+ }
+}
+
+var didothers bool
+var deadlock mutex
+
+// gp is the crashing g running on this M, but may be a user G, while getg() is
+// always g0.
+func dopanic_m(gp *g, pc, sp uintptr) bool {
+ if gp.sig != 0 {
+ signame := signame(gp.sig)
+ if signame != "" {
+ print("[signal ", signame)
+ } else {
+ print("[signal ", hex(gp.sig))
+ }
+ print(" code=", hex(gp.sigcode0), " addr=", hex(gp.sigcode1), " pc=", hex(gp.sigpc), "]\n")
+ }
+
+ level, all, docrash := gotraceback()
+ if level > 0 {
+ if gp != gp.m.curg {
+ all = true
+ }
+ if gp != gp.m.g0 {
+ print("\n")
+ goroutineheader(gp)
+ traceback(pc, sp, 0, gp)
+ } else if level >= 2 || gp.m.throwing >= throwTypeRuntime {
+ print("\nruntime stack:\n")
+ traceback(pc, sp, 0, gp)
+ }
+ if !didothers && all {
+ didothers = true
+ tracebackothers(gp)
+ }
+ }
+ unlock(&paniclk)
+
+ if panicking.Add(-1) != 0 {
+ // Some other m is panicking too.
+ // Let it print what it needs to print.
+ // Wait forever without chewing up cpu.
+ // It will exit when it's done.
+ lock(&deadlock)
+ lock(&deadlock)
+ }
+
+ printDebugLog()
+
+ return docrash
+}
+
+// canpanic returns false if a signal should throw instead of
+// panicking.
+//
+//go:nosplit
+func canpanic() bool {
+ gp := getg()
+ mp := acquirem()
+
+ // Is it okay for gp to panic instead of crashing the program?
+ // Yes, as long as it is running Go code, not runtime code,
+ // and not stuck in a system call.
+ if gp != mp.curg {
+ releasem(mp)
+ return false
+ }
+ // N.B. mp.locks != 1 instead of 0 to account for acquirem.
+ if mp.locks != 1 || mp.mallocing != 0 || mp.throwing != throwTypeNone || mp.preemptoff != "" || mp.dying != 0 {
+ releasem(mp)
+ return false
+ }
+ status := readgstatus(gp)
+ if status&^_Gscan != _Grunning || gp.syscallsp != 0 {
+ releasem(mp)
+ return false
+ }
+ if GOOS == "windows" && mp.libcallsp != 0 {
+ releasem(mp)
+ return false
+ }
+ releasem(mp)
+ return true
+}
+
+// shouldPushSigpanic reports whether pc should be used as sigpanic's
+// return PC (pushing a frame for the call). Otherwise, it should be
+// left alone so that LR is used as sigpanic's return PC, effectively
+// replacing the top-most frame with sigpanic. This is used by
+// preparePanic.
+func shouldPushSigpanic(gp *g, pc, lr uintptr) bool {
+ if pc == 0 {
+ // Probably a call to a nil func. The old LR is more
+ // useful in the stack trace. Not pushing the frame
+ // will make the trace look like a call to sigpanic
+ // instead. (Otherwise the trace will end at sigpanic
+ // and we won't get to see who faulted.)
+ return false
+ }
+ // If we don't recognize the PC as code, but we do recognize
+ // the link register as code, then this assumes the panic was
+ // caused by a call to non-code. In this case, we want to
+ // ignore this call to make unwinding show the context.
+ //
+ // If we running C code, we're not going to recognize pc as a
+ // Go function, so just assume it's good. Otherwise, traceback
+ // may try to read a stale LR that looks like a Go code
+ // pointer and wander into the woods.
+ if gp.m.incgo || findfunc(pc).valid() {
+ // This wasn't a bad call, so use PC as sigpanic's
+ // return PC.
+ return true
+ }
+ if findfunc(lr).valid() {
+ // This was a bad call, but the LR is good, so use the
+ // LR as sigpanic's return PC.
+ return false
+ }
+ // Neither the PC or LR is good. Hopefully pushing a frame
+ // will work.
+ return true
+}
+
+// isAbortPC reports whether pc is the program counter at which
+// runtime.abort raises a signal.
+//
+// It is nosplit because it's part of the isgoexception
+// implementation.
+//
+//go:nosplit
+func isAbortPC(pc uintptr) bool {
+ f := findfunc(pc)
+ if !f.valid() {
+ return false
+ }
+ return f.funcID == abi.FuncID_abort
+}