diff options
Diffstat (limited to 'src/runtime/mbarrier.go')
| -rw-r--r-- | src/runtime/mbarrier.go | 373 |
1 files changed, 373 insertions, 0 deletions
diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go new file mode 100644 index 0000000..c4b6c2a --- /dev/null +++ b/src/runtime/mbarrier.go @@ -0,0 +1,373 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: write barriers. +// +// For the concurrent garbage collector, the Go compiler implements +// updates to pointer-valued fields that may be in heap objects by +// emitting calls to write barriers. The main write barrier for +// individual pointer writes is gcWriteBarrier and is implemented in +// assembly. This file contains write barrier entry points for bulk +// operations. See also mwbbuf.go. + +package runtime + +import ( + "internal/abi" + "internal/goarch" + "internal/goexperiment" + "unsafe" +) + +// Go uses a hybrid barrier that combines a Yuasa-style deletion +// barrier—which shades the object whose reference is being +// overwritten—with Dijkstra insertion barrier—which shades the object +// whose reference is being written. The insertion part of the barrier +// is necessary while the calling goroutine's stack is grey. In +// pseudocode, the barrier is: +// +// writePointer(slot, ptr): +// shade(*slot) +// if current stack is grey: +// shade(ptr) +// *slot = ptr +// +// slot is the destination in Go code. +// ptr is the value that goes into the slot in Go code. +// +// Shade indicates that it has seen a white pointer by adding the referent +// to wbuf as well as marking it. +// +// The two shades and the condition work together to prevent a mutator +// from hiding an object from the garbage collector: +// +// 1. shade(*slot) prevents a mutator from hiding an object by moving +// the sole pointer to it from the heap to its stack. If it attempts +// to unlink an object from the heap, this will shade it. +// +// 2. shade(ptr) prevents a mutator from hiding an object by moving +// the sole pointer to it from its stack into a black object in the +// heap. If it attempts to install the pointer into a black object, +// this will shade it. +// +// 3. Once a goroutine's stack is black, the shade(ptr) becomes +// unnecessary. shade(ptr) prevents hiding an object by moving it from +// the stack to the heap, but this requires first having a pointer +// hidden on the stack. Immediately after a stack is scanned, it only +// points to shaded objects, so it's not hiding anything, and the +// shade(*slot) prevents it from hiding any other pointers on its +// stack. +// +// For a detailed description of this barrier and proof of +// correctness, see https://github.com/golang/proposal/blob/master/design/17503-eliminate-rescan.md +// +// +// +// Dealing with memory ordering: +// +// Both the Yuasa and Dijkstra barriers can be made conditional on the +// color of the object containing the slot. We chose not to make these +// conditional because the cost of ensuring that the object holding +// the slot doesn't concurrently change color without the mutator +// noticing seems prohibitive. +// +// Consider the following example where the mutator writes into +// a slot and then loads the slot's mark bit while the GC thread +// writes to the slot's mark bit and then as part of scanning reads +// the slot. +// +// Initially both [slot] and [slotmark] are 0 (nil) +// Mutator thread GC thread +// st [slot], ptr st [slotmark], 1 +// +// ld r1, [slotmark] ld r2, [slot] +// +// Without an expensive memory barrier between the st and the ld, the final +// result on most HW (including 386/amd64) can be r1==r2==0. This is a classic +// example of what can happen when loads are allowed to be reordered with older +// stores (avoiding such reorderings lies at the heart of the classic +// Peterson/Dekker algorithms for mutual exclusion). Rather than require memory +// barriers, which will slow down both the mutator and the GC, we always grey +// the ptr object regardless of the slot's color. +// +// Another place where we intentionally omit memory barriers is when +// accessing mheap_.arena_used to check if a pointer points into the +// heap. On relaxed memory machines, it's possible for a mutator to +// extend the size of the heap by updating arena_used, allocate an +// object from this new region, and publish a pointer to that object, +// but for tracing running on another processor to observe the pointer +// but use the old value of arena_used. In this case, tracing will not +// mark the object, even though it's reachable. However, the mutator +// is guaranteed to execute a write barrier when it publishes the +// pointer, so it will take care of marking the object. A general +// consequence of this is that the garbage collector may cache the +// value of mheap_.arena_used. (See issue #9984.) +// +// +// Stack writes: +// +// The compiler omits write barriers for writes to the current frame, +// but if a stack pointer has been passed down the call stack, the +// compiler will generate a write barrier for writes through that +// pointer (because it doesn't know it's not a heap pointer). +// +// +// Global writes: +// +// The Go garbage collector requires write barriers when heap pointers +// are stored in globals. Many garbage collectors ignore writes to +// globals and instead pick up global -> heap pointers during +// termination. This increases pause time, so we instead rely on write +// barriers for writes to globals so that we don't have to rescan +// global during mark termination. +// +// +// Publication ordering: +// +// The write barrier is *pre-publication*, meaning that the write +// barrier happens prior to the *slot = ptr write that may make ptr +// reachable by some goroutine that currently cannot reach it. +// +// +// Signal handler pointer writes: +// +// In general, the signal handler cannot safely invoke the write +// barrier because it may run without a P or even during the write +// barrier. +// +// There is exactly one exception: profbuf.go omits a barrier during +// signal handler profile logging. That's safe only because of the +// deletion barrier. See profbuf.go for a detailed argument. If we +// remove the deletion barrier, we'll have to work out a new way to +// handle the profile logging. + +// typedmemmove copies a value of type typ to dst from src. +// Must be nosplit, see #16026. +// +// TODO: Perfect for go:nosplitrec since we can't have a safe point +// anywhere in the bulk barrier or memmove. +// +//go:nosplit +func typedmemmove(typ *abi.Type, dst, src unsafe.Pointer) { + if dst == src { + return + } + if writeBarrier.enabled && typ.PtrBytes != 0 { + // This always copies a full value of type typ so it's safe + // to pass typ along as an optimization. See the comment on + // bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes, typ) + } + // There's a race here: if some other goroutine can write to + // src, it may change some pointer in src after we've + // performed the write barrier but before we perform the + // memory copy. This safe because the write performed by that + // other goroutine must also be accompanied by a write + // barrier, so at worst we've unnecessarily greyed the old + // pointer that was in src. + memmove(dst, src, typ.Size_) + if goexperiment.CgoCheck2 { + cgoCheckMemmove2(typ, dst, src, 0, typ.Size_) + } +} + +// wbZero performs the write barrier operations necessary before +// zeroing a region of memory at address dst of type typ. +// Does not actually do the zeroing. +// +//go:nowritebarrierrec +//go:nosplit +func wbZero(typ *_type, dst unsafe.Pointer) { + // This always copies a full value of type typ so it's safe + // to pass typ along as an optimization. See the comment on + // bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(dst), 0, typ.PtrBytes, typ) +} + +// wbMove performs the write barrier operations necessary before +// copying a region of memory from src to dst of type typ. +// Does not actually do the copying. +// +//go:nowritebarrierrec +//go:nosplit +func wbMove(typ *_type, dst, src unsafe.Pointer) { + // This always copies a full value of type typ so it's safe to + // pass a type here. + // + // See the comment on bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.PtrBytes, typ) +} + +//go:linkname reflect_typedmemmove reflect.typedmemmove +func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + if raceenabled { + raceWriteObjectPC(typ, dst, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + raceReadObjectPC(typ, src, getcallerpc(), abi.FuncPCABIInternal(reflect_typedmemmove)) + } + if msanenabled { + msanwrite(dst, typ.Size_) + msanread(src, typ.Size_) + } + if asanenabled { + asanwrite(dst, typ.Size_) + asanread(src, typ.Size_) + } + typedmemmove(typ, dst, src) +} + +//go:linkname reflectlite_typedmemmove internal/reflectlite.typedmemmove +func reflectlite_typedmemmove(typ *_type, dst, src unsafe.Pointer) { + reflect_typedmemmove(typ, dst, src) +} + +// reflectcallmove is invoked by reflectcall to copy the return values +// out of the stack and into the heap, invoking the necessary write +// barriers. dst, src, and size describe the return value area to +// copy. typ describes the entire frame (not just the return values). +// typ may be nil, which indicates write barriers are not needed. +// +// It must be nosplit and must only call nosplit functions because the +// stack map of reflectcall is wrong. +// +//go:nosplit +func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr, regs *abi.RegArgs) { + if writeBarrier.enabled && typ != nil && typ.PtrBytes != 0 && size >= goarch.PtrSize { + // Pass nil for the type. dst does not point to value of type typ, + // but rather points into one, so applying the optimization is not + // safe. See the comment on this function. + bulkBarrierPreWrite(uintptr(dst), uintptr(src), size, nil) + } + memmove(dst, src, size) + + // Move pointers returned in registers to a place where the GC can see them. + for i := range regs.Ints { + if regs.ReturnIsPtr.Get(i) { + regs.Ptrs[i] = unsafe.Pointer(regs.Ints[i]) + } + } +} + +//go:nosplit +func typedslicecopy(typ *_type, dstPtr unsafe.Pointer, dstLen int, srcPtr unsafe.Pointer, srcLen int) int { + n := dstLen + if n > srcLen { + n = srcLen + } + if n == 0 { + return 0 + } + + // The compiler emits calls to typedslicecopy before + // instrumentation runs, so unlike the other copying and + // assignment operations, it's not instrumented in the calling + // code and needs its own instrumentation. + if raceenabled { + callerpc := getcallerpc() + pc := abi.FuncPCABIInternal(slicecopy) + racewriterangepc(dstPtr, uintptr(n)*typ.Size_, callerpc, pc) + racereadrangepc(srcPtr, uintptr(n)*typ.Size_, callerpc, pc) + } + if msanenabled { + msanwrite(dstPtr, uintptr(n)*typ.Size_) + msanread(srcPtr, uintptr(n)*typ.Size_) + } + if asanenabled { + asanwrite(dstPtr, uintptr(n)*typ.Size_) + asanread(srcPtr, uintptr(n)*typ.Size_) + } + + if goexperiment.CgoCheck2 { + cgoCheckSliceCopy(typ, dstPtr, srcPtr, n) + } + + if dstPtr == srcPtr { + return n + } + + // Note: No point in checking typ.PtrBytes here: + // compiler only emits calls to typedslicecopy for types with pointers, + // and growslice and reflect_typedslicecopy check for pointers + // before calling typedslicecopy. + size := uintptr(n) * typ.Size_ + if writeBarrier.enabled { + // This always copies one or more full values of type typ so + // it's safe to pass typ along as an optimization. See the comment on + // bulkBarrierPreWrite. + pwsize := size - typ.Size_ + typ.PtrBytes + bulkBarrierPreWrite(uintptr(dstPtr), uintptr(srcPtr), pwsize, typ) + } + // See typedmemmove for a discussion of the race between the + // barrier and memmove. + memmove(dstPtr, srcPtr, size) + return n +} + +//go:linkname reflect_typedslicecopy reflect.typedslicecopy +func reflect_typedslicecopy(elemType *_type, dst, src slice) int { + if elemType.PtrBytes == 0 { + return slicecopy(dst.array, dst.len, src.array, src.len, elemType.Size_) + } + return typedslicecopy(elemType, dst.array, dst.len, src.array, src.len) +} + +// typedmemclr clears the typed memory at ptr with type typ. The +// memory at ptr must already be initialized (and hence in type-safe +// state). If the memory is being initialized for the first time, see +// memclrNoHeapPointers. +// +// If the caller knows that typ has pointers, it can alternatively +// call memclrHasPointers. +// +// TODO: A "go:nosplitrec" annotation would be perfect for this. +// +//go:nosplit +func typedmemclr(typ *_type, ptr unsafe.Pointer) { + if writeBarrier.enabled && typ.PtrBytes != 0 { + // This always clears a whole value of type typ, so it's + // safe to pass a type here and apply the optimization. + // See the comment on bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(ptr), 0, typ.PtrBytes, typ) + } + memclrNoHeapPointers(ptr, typ.Size_) +} + +//go:linkname reflect_typedmemclr reflect.typedmemclr +func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + +//go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial +func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { + if writeBarrier.enabled && typ.PtrBytes != 0 { + // Pass nil for the type. ptr does not point to value of type typ, + // but rather points into one so it's not safe to apply the optimization. + // See the comment on this function in the reflect package and the + // comment on bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(ptr), 0, size, nil) + } + memclrNoHeapPointers(ptr, size) +} + +//go:linkname reflect_typedarrayclear reflect.typedarrayclear +func reflect_typedarrayclear(typ *_type, ptr unsafe.Pointer, len int) { + size := typ.Size_ * uintptr(len) + if writeBarrier.enabled && typ.PtrBytes != 0 { + // This always clears whole elements of an array, so it's + // safe to pass a type here. See the comment on bulkBarrierPreWrite. + bulkBarrierPreWrite(uintptr(ptr), 0, size, typ) + } + memclrNoHeapPointers(ptr, size) +} + +// memclrHasPointers clears n bytes of typed memory starting at ptr. +// The caller must ensure that the type of the object at ptr has +// pointers, usually by checking typ.PtrBytes. However, ptr +// does not have to point to the start of the allocation. +// +//go:nosplit +func memclrHasPointers(ptr unsafe.Pointer, n uintptr) { + // Pass nil for the type since we don't have one here anyway. + bulkBarrierPreWrite(uintptr(ptr), 0, n, nil) + memclrNoHeapPointers(ptr, n) +} |