diff options
Diffstat (limited to '')
93 files changed, 6807 insertions, 0 deletions
diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go new file mode 100644 index 0000000..d904682 --- /dev/null +++ b/src/runtime/cgo.go @@ -0,0 +1,54 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:cgo_export_static main + +// Filled in by runtime/cgo when linked into binary. + +//go:linkname _cgo_init _cgo_init +//go:linkname _cgo_thread_start _cgo_thread_start +//go:linkname _cgo_sys_thread_create _cgo_sys_thread_create +//go:linkname _cgo_notify_runtime_init_done _cgo_notify_runtime_init_done +//go:linkname _cgo_callers _cgo_callers +//go:linkname _cgo_set_context_function _cgo_set_context_function +//go:linkname _cgo_yield _cgo_yield + +var ( + _cgo_init unsafe.Pointer + _cgo_thread_start unsafe.Pointer + _cgo_sys_thread_create unsafe.Pointer + _cgo_notify_runtime_init_done unsafe.Pointer + _cgo_callers unsafe.Pointer + _cgo_set_context_function unsafe.Pointer + _cgo_yield unsafe.Pointer +) + +// iscgo is set to true by the runtime/cgo package +var iscgo bool + +// cgoHasExtraM is set on startup when an extra M is created for cgo. +// The extra M must be created before any C/C++ code calls cgocallback. +var cgoHasExtraM bool + +// cgoUse is called by cgo-generated code (using go:linkname to get at +// an unexported name). The calls serve two purposes: +// 1) they are opaque to escape analysis, so the argument is considered to +// escape to the heap. +// 2) they keep the argument alive until the call site; the call is emitted after +// the end of the (presumed) use of the argument by C. +// cgoUse should not actually be called (see cgoAlwaysFalse). +func cgoUse(any) { throw("cgoUse should not be called") } + +// cgoAlwaysFalse is a boolean value that is always false. +// The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }. +// The compiler cannot see that cgoAlwaysFalse is always false, +// so it emits the test and keeps the call, giving the desired +// escape analysis result. The test is cheaper than the call. +var cgoAlwaysFalse bool + +var cgo_yield = &_cgo_yield diff --git a/src/runtime/cgo/abi_amd64.h b/src/runtime/cgo/abi_amd64.h new file mode 100644 index 0000000..9949435 --- /dev/null +++ b/src/runtime/cgo/abi_amd64.h @@ -0,0 +1,99 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Macros for transitioning from the host ABI to Go ABI0. +// +// These save the frame pointer, so in general, functions that use +// these should have zero frame size to suppress the automatic frame +// pointer, though it's harmless to not do this. + +#ifdef GOOS_windows + +// REGS_HOST_TO_ABI0_STACK is the stack bytes used by +// PUSH_REGS_HOST_TO_ABI0. +#define REGS_HOST_TO_ABI0_STACK (28*8 + 8) + +// PUSH_REGS_HOST_TO_ABI0 prepares for transitioning from +// the host ABI to Go ABI0 code. It saves all registers that are +// callee-save in the host ABI and caller-save in Go ABI0 and prepares +// for entry to Go. +// +// Save DI SI BP BX R12 R13 R14 R15 X6-X15 registers and the DF flag. +// Clear the DF flag for the Go ABI. +// MXCSR matches the Go ABI, so we don't have to set that, +// and Go doesn't modify it, so we don't have to save it. +#define PUSH_REGS_HOST_TO_ABI0() \ + PUSHFQ \ + CLD \ + ADJSP $(REGS_HOST_TO_ABI0_STACK - 8) \ + MOVQ DI, (0*0)(SP) \ + MOVQ SI, (1*8)(SP) \ + MOVQ BP, (2*8)(SP) \ + MOVQ BX, (3*8)(SP) \ + MOVQ R12, (4*8)(SP) \ + MOVQ R13, (5*8)(SP) \ + MOVQ R14, (6*8)(SP) \ + MOVQ R15, (7*8)(SP) \ + MOVUPS X6, (8*8)(SP) \ + MOVUPS X7, (10*8)(SP) \ + MOVUPS X8, (12*8)(SP) \ + MOVUPS X9, (14*8)(SP) \ + MOVUPS X10, (16*8)(SP) \ + MOVUPS X11, (18*8)(SP) \ + MOVUPS X12, (20*8)(SP) \ + MOVUPS X13, (22*8)(SP) \ + MOVUPS X14, (24*8)(SP) \ + MOVUPS X15, (26*8)(SP) + +#define POP_REGS_HOST_TO_ABI0() \ + MOVQ (0*0)(SP), DI \ + MOVQ (1*8)(SP), SI \ + MOVQ (2*8)(SP), BP \ + MOVQ (3*8)(SP), BX \ + MOVQ (4*8)(SP), R12 \ + MOVQ (5*8)(SP), R13 \ + MOVQ (6*8)(SP), R14 \ + MOVQ (7*8)(SP), R15 \ + MOVUPS (8*8)(SP), X6 \ + MOVUPS (10*8)(SP), X7 \ + MOVUPS (12*8)(SP), X8 \ + MOVUPS (14*8)(SP), X9 \ + MOVUPS (16*8)(SP), X10 \ + MOVUPS (18*8)(SP), X11 \ + MOVUPS (20*8)(SP), X12 \ + MOVUPS (22*8)(SP), X13 \ + MOVUPS (24*8)(SP), X14 \ + MOVUPS (26*8)(SP), X15 \ + ADJSP $-(REGS_HOST_TO_ABI0_STACK - 8) \ + POPFQ + +#else +// SysV ABI + +#define REGS_HOST_TO_ABI0_STACK (6*8) + +// SysV MXCSR matches the Go ABI, so we don't have to set that, +// and Go doesn't modify it, so we don't have to save it. +// Both SysV and Go require DF to be cleared, so that's already clear. +// The SysV and Go frame pointer conventions are compatible. +#define PUSH_REGS_HOST_TO_ABI0() \ + ADJSP $(REGS_HOST_TO_ABI0_STACK) \ + MOVQ BP, (5*8)(SP) \ + LEAQ (5*8)(SP), BP \ + MOVQ BX, (0*8)(SP) \ + MOVQ R12, (1*8)(SP) \ + MOVQ R13, (2*8)(SP) \ + MOVQ R14, (3*8)(SP) \ + MOVQ R15, (4*8)(SP) + +#define POP_REGS_HOST_TO_ABI0() \ + MOVQ (0*8)(SP), BX \ + MOVQ (1*8)(SP), R12 \ + MOVQ (2*8)(SP), R13 \ + MOVQ (3*8)(SP), R14 \ + MOVQ (4*8)(SP), R15 \ + MOVQ (5*8)(SP), BP \ + ADJSP $-(REGS_HOST_TO_ABI0_STACK) + +#endif diff --git a/src/runtime/cgo/asm_386.s b/src/runtime/cgo/asm_386.s new file mode 100644 index 0000000..2e7e951 --- /dev/null +++ b/src/runtime/cgo/asm_386.s @@ -0,0 +1,29 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT,$28-16 + MOVL BP, 24(SP) + MOVL BX, 20(SP) + MOVL SI, 16(SP) + MOVL DI, 12(SP) + + MOVL ctxt+12(FP), AX + MOVL AX, 8(SP) + MOVL a+4(FP), AX + MOVL AX, 4(SP) + MOVL fn+0(FP), AX + MOVL AX, 0(SP) + CALL runtime·cgocallback(SB) + + MOVL 12(SP), DI + MOVL 16(SP), SI + MOVL 20(SP), BX + MOVL 24(SP), BP + RET diff --git a/src/runtime/cgo/asm_amd64.s b/src/runtime/cgo/asm_amd64.s new file mode 100644 index 0000000..386299c --- /dev/null +++ b/src/runtime/cgo/asm_amd64.s @@ -0,0 +1,34 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" +#include "abi_amd64.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +// This signature is known to SWIG, so we can't change it. +TEXT crosscall2(SB),NOSPLIT,$0-0 + PUSH_REGS_HOST_TO_ABI0() + + // Make room for arguments to cgocallback. + ADJSP $0x18 +#ifndef GOOS_windows + MOVQ DI, 0x0(SP) /* fn */ + MOVQ SI, 0x8(SP) /* arg */ + // Skip n in DX. + MOVQ CX, 0x10(SP) /* ctxt */ +#else + MOVQ CX, 0x0(SP) /* fn */ + MOVQ DX, 0x8(SP) /* arg */ + // Skip n in R8. + MOVQ R9, 0x10(SP) /* ctxt */ +#endif + + CALL runtime·cgocallback(SB) + + ADJSP $-0x18 + POP_REGS_HOST_TO_ABI0() + RET diff --git a/src/runtime/cgo/asm_arm.s b/src/runtime/cgo/asm_arm.s new file mode 100644 index 0000000..ea55e17 --- /dev/null +++ b/src/runtime/cgo/asm_arm.s @@ -0,0 +1,56 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + SUB $(8*9), R13 // Reserve space for the floating point registers. + // The C arguments arrive in R0, R1, R2, and R3. We want to + // pass R0, R1, and R3 to Go, so we push those on the stack. + // Also, save C callee-save registers R4-R12. + MOVM.WP [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12], (R13) + // Finally, save the link register R14. This also puts the + // arguments we pushed for cgocallback where they need to be, + // starting at 4(R13). + MOVW.W R14, -4(R13) + + // Skip floating point registers on GOARM < 6. + MOVB runtime·goarm(SB), R11 + CMP $6, R11 + BLT skipfpsave + MOVD F8, (13*4+8*1)(R13) + MOVD F9, (13*4+8*2)(R13) + MOVD F10, (13*4+8*3)(R13) + MOVD F11, (13*4+8*4)(R13) + MOVD F12, (13*4+8*5)(R13) + MOVD F13, (13*4+8*6)(R13) + MOVD F14, (13*4+8*7)(R13) + MOVD F15, (13*4+8*8)(R13) + +skipfpsave: + BL runtime·load_g(SB) + // We set up the arguments to cgocallback when saving registers above. + BL runtime·cgocallback(SB) + + MOVB runtime·goarm(SB), R11 + CMP $6, R11 + BLT skipfprest + MOVD (13*4+8*1)(R13), F8 + MOVD (13*4+8*2)(R13), F9 + MOVD (13*4+8*3)(R13), F10 + MOVD (13*4+8*4)(R13), F11 + MOVD (13*4+8*5)(R13), F12 + MOVD (13*4+8*6)(R13), F13 + MOVD (13*4+8*7)(R13), F14 + MOVD (13*4+8*8)(R13), F15 + +skipfprest: + MOVW.P 4(R13), R14 + MOVM.IAW (R13), [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12] + ADD $(8*9), R13 + MOVW R14, R15 diff --git a/src/runtime/cgo/asm_arm64.s b/src/runtime/cgo/asm_arm64.s new file mode 100644 index 0000000..1cb25cf --- /dev/null +++ b/src/runtime/cgo/asm_arm64.s @@ -0,0 +1,70 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + /* + * We still need to save all callee save register as before, and then + * push 3 args for fn (R0, R1, R3), skipping R2. + * Also note that at procedure entry in gc world, 8(RSP) will be the + * first arg. + * TODO(minux): use LDP/STP here if it matters. + */ + SUB $(8*24), RSP + MOVD R0, (8*1)(RSP) + MOVD R1, (8*2)(RSP) + MOVD R3, (8*3)(RSP) + MOVD R19, (8*4)(RSP) + MOVD R20, (8*5)(RSP) + MOVD R21, (8*6)(RSP) + MOVD R22, (8*7)(RSP) + MOVD R23, (8*8)(RSP) + MOVD R24, (8*9)(RSP) + MOVD R25, (8*10)(RSP) + MOVD R26, (8*11)(RSP) + MOVD R27, (8*12)(RSP) + MOVD g, (8*13)(RSP) + MOVD R29, (8*14)(RSP) + MOVD R30, (8*15)(RSP) + FMOVD F8, (8*16)(RSP) + FMOVD F9, (8*17)(RSP) + FMOVD F10, (8*18)(RSP) + FMOVD F11, (8*19)(RSP) + FMOVD F12, (8*20)(RSP) + FMOVD F13, (8*21)(RSP) + FMOVD F14, (8*22)(RSP) + FMOVD F15, (8*23)(RSP) + + // Initialize Go ABI environment + BL runtime·load_g(SB) + + BL runtime·cgocallback(SB) + + MOVD (8*4)(RSP), R19 + MOVD (8*5)(RSP), R20 + MOVD (8*6)(RSP), R21 + MOVD (8*7)(RSP), R22 + MOVD (8*8)(RSP), R23 + MOVD (8*9)(RSP), R24 + MOVD (8*10)(RSP), R25 + MOVD (8*11)(RSP), R26 + MOVD (8*12)(RSP), R27 + MOVD (8*13)(RSP), g + MOVD (8*14)(RSP), R29 + MOVD (8*15)(RSP), R30 + FMOVD (8*16)(RSP), F8 + FMOVD (8*17)(RSP), F9 + FMOVD (8*18)(RSP), F10 + FMOVD (8*19)(RSP), F11 + FMOVD (8*20)(RSP), F12 + FMOVD (8*21)(RSP), F13 + FMOVD (8*22)(RSP), F14 + FMOVD (8*23)(RSP), F15 + ADD $(8*24), RSP + RET diff --git a/src/runtime/cgo/asm_mips64x.s b/src/runtime/cgo/asm_mips64x.s new file mode 100644 index 0000000..ba94807 --- /dev/null +++ b/src/runtime/cgo/asm_mips64x.s @@ -0,0 +1,83 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips64 || mips64le +// +build mips64 mips64le + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + /* + * We still need to save all callee save register as before, and then + * push 3 args for fn (R4, R5, R7), skipping R6. + * Also note that at procedure entry in gc world, 8(R29) will be the + * first arg. + */ +#ifndef GOMIPS64_softfloat + ADDV $(-8*23), R29 +#else + ADDV $(-8*15), R29 +#endif + MOVV R4, (8*1)(R29) // fn unsafe.Pointer + MOVV R5, (8*2)(R29) // a unsafe.Pointer + MOVV R7, (8*3)(R29) // ctxt uintptr + MOVV R16, (8*4)(R29) + MOVV R17, (8*5)(R29) + MOVV R18, (8*6)(R29) + MOVV R19, (8*7)(R29) + MOVV R20, (8*8)(R29) + MOVV R21, (8*9)(R29) + MOVV R22, (8*10)(R29) + MOVV R23, (8*11)(R29) + MOVV RSB, (8*12)(R29) + MOVV g, (8*13)(R29) + MOVV R31, (8*14)(R29) +#ifndef GOMIPS64_softfloat + MOVD F24, (8*15)(R29) + MOVD F25, (8*16)(R29) + MOVD F26, (8*17)(R29) + MOVD F27, (8*18)(R29) + MOVD F28, (8*19)(R29) + MOVD F29, (8*20)(R29) + MOVD F30, (8*21)(R29) + MOVD F31, (8*22)(R29) +#endif + // Initialize Go ABI environment + // prepare SB register = PC & 0xffffffff00000000 + BGEZAL R0, 1(PC) + SRLV $32, R31, RSB + SLLV $32, RSB + JAL runtime·load_g(SB) + + JAL runtime·cgocallback(SB) + + MOVV (8*4)(R29), R16 + MOVV (8*5)(R29), R17 + MOVV (8*6)(R29), R18 + MOVV (8*7)(R29), R19 + MOVV (8*8)(R29), R20 + MOVV (8*9)(R29), R21 + MOVV (8*10)(R29), R22 + MOVV (8*11)(R29), R23 + MOVV (8*12)(R29), RSB + MOVV (8*13)(R29), g + MOVV (8*14)(R29), R31 +#ifndef GOMIPS64_softfloat + MOVD (8*15)(R29), F24 + MOVD (8*16)(R29), F25 + MOVD (8*17)(R29), F26 + MOVD (8*18)(R29), F27 + MOVD (8*19)(R29), F28 + MOVD (8*20)(R29), F29 + MOVD (8*21)(R29), F30 + MOVD (8*22)(R29), F31 + ADDV $(8*23), R29 +#else + ADDV $(8*15), R29 +#endif + RET diff --git a/src/runtime/cgo/asm_mipsx.s b/src/runtime/cgo/asm_mipsx.s new file mode 100644 index 0000000..fd5d78e --- /dev/null +++ b/src/runtime/cgo/asm_mipsx.s @@ -0,0 +1,76 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build mips || mipsle +// +build mips mipsle + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + /* + * We still need to save all callee save register as before, and then + * push 3 args for fn (R4, R5, R7), skipping R6. + * Also note that at procedure entry in gc world, 4(R29) will be the + * first arg. + */ + + // Space for 9 caller-saved GPR + LR + 6 caller-saved FPR. + // O32 ABI allows us to smash 16 bytes argument area of caller frame. +#ifndef GOMIPS_softfloat + SUBU $(4*14+8*6-16), R29 +#else + SUBU $(4*14-16), R29 // For soft-float, no FPR. +#endif + MOVW R4, (4*1)(R29) // fn unsafe.Pointer + MOVW R5, (4*2)(R29) // a unsafe.Pointer + MOVW R7, (4*3)(R29) // ctxt uintptr + MOVW R16, (4*4)(R29) + MOVW R17, (4*5)(R29) + MOVW R18, (4*6)(R29) + MOVW R19, (4*7)(R29) + MOVW R20, (4*8)(R29) + MOVW R21, (4*9)(R29) + MOVW R22, (4*10)(R29) + MOVW R23, (4*11)(R29) + MOVW g, (4*12)(R29) + MOVW R31, (4*13)(R29) +#ifndef GOMIPS_softfloat + MOVD F20, (4*14)(R29) + MOVD F22, (4*14+8*1)(R29) + MOVD F24, (4*14+8*2)(R29) + MOVD F26, (4*14+8*3)(R29) + MOVD F28, (4*14+8*4)(R29) + MOVD F30, (4*14+8*5)(R29) +#endif + JAL runtime·load_g(SB) + + JAL runtime·cgocallback(SB) + + MOVW (4*4)(R29), R16 + MOVW (4*5)(R29), R17 + MOVW (4*6)(R29), R18 + MOVW (4*7)(R29), R19 + MOVW (4*8)(R29), R20 + MOVW (4*9)(R29), R21 + MOVW (4*10)(R29), R22 + MOVW (4*11)(R29), R23 + MOVW (4*12)(R29), g + MOVW (4*13)(R29), R31 +#ifndef GOMIPS_softfloat + MOVD (4*14)(R29), F20 + MOVD (4*14+8*1)(R29), F22 + MOVD (4*14+8*2)(R29), F24 + MOVD (4*14+8*3)(R29), F26 + MOVD (4*14+8*4)(R29), F28 + MOVD (4*14+8*5)(R29), F30 + + ADDU $(4*14+8*6-16), R29 +#else + ADDU $(4*14-16), R29 +#endif + RET diff --git a/src/runtime/cgo/asm_ppc64x.s b/src/runtime/cgo/asm_ppc64x.s new file mode 100644 index 0000000..187b2d4 --- /dev/null +++ b/src/runtime/cgo/asm_ppc64x.s @@ -0,0 +1,139 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le +// +build ppc64 ppc64le + +#include "textflag.h" +#include "asm_ppc64x.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +// The value of R2 is saved on the new stack frame, and not +// the caller's frame due to issue #43228. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + // Start with standard C stack frame layout and linkage + MOVD LR, R0 + MOVD R0, 16(R1) // Save LR in caller's frame + MOVW CR, R0 // Save CR in caller's frame + MOVW R0, 8(R1) + + BL saveregs2<>(SB) + + MOVDU R1, (-288-3*8-FIXED_FRAME)(R1) + // Save the caller's R2 + MOVD R2, 24(R1) + + // Initialize Go ABI environment + BL runtime·reginit(SB) + BL runtime·load_g(SB) + +#ifdef GOARCH_ppc64 + // ppc64 use elf ABI v1. we must get the real entry address from + // first slot of the function descriptor before call. + // Same for AIX. + MOVD 8(R3), R2 + MOVD (R3), R3 +#endif + MOVD R3, FIXED_FRAME+0(R1) // fn unsafe.Pointer + MOVD R4, FIXED_FRAME+8(R1) // a unsafe.Pointer + // Skip R5 = n uint32 + MOVD R6, FIXED_FRAME+16(R1) // ctxt uintptr + BL runtime·cgocallback(SB) + + // Restore the caller's R2 + MOVD 24(R1), R2 + ADD $(288+3*8+FIXED_FRAME), R1 + + BL restoreregs2<>(SB) + + MOVW 8(R1), R0 + MOVFL R0, $0xff + MOVD 16(R1), R0 + MOVD R0, LR + RET + +TEXT saveregs2<>(SB),NOSPLIT|NOFRAME,$0 + // O=-288; for R in R{14..31}; do echo "\tMOVD\t$R, $O(R1)"|sed s/R30/g/; ((O+=8)); done; for F in F{14..31}; do echo "\tFMOVD\t$F, $O(R1)"; ((O+=8)); done + MOVD R14, -288(R1) + MOVD R15, -280(R1) + MOVD R16, -272(R1) + MOVD R17, -264(R1) + MOVD R18, -256(R1) + MOVD R19, -248(R1) + MOVD R20, -240(R1) + MOVD R21, -232(R1) + MOVD R22, -224(R1) + MOVD R23, -216(R1) + MOVD R24, -208(R1) + MOVD R25, -200(R1) + MOVD R26, -192(R1) + MOVD R27, -184(R1) + MOVD R28, -176(R1) + MOVD R29, -168(R1) + MOVD g, -160(R1) + MOVD R31, -152(R1) + FMOVD F14, -144(R1) + FMOVD F15, -136(R1) + FMOVD F16, -128(R1) + FMOVD F17, -120(R1) + FMOVD F18, -112(R1) + FMOVD F19, -104(R1) + FMOVD F20, -96(R1) + FMOVD F21, -88(R1) + FMOVD F22, -80(R1) + FMOVD F23, -72(R1) + FMOVD F24, -64(R1) + FMOVD F25, -56(R1) + FMOVD F26, -48(R1) + FMOVD F27, -40(R1) + FMOVD F28, -32(R1) + FMOVD F29, -24(R1) + FMOVD F30, -16(R1) + FMOVD F31, -8(R1) + + RET + +TEXT restoreregs2<>(SB),NOSPLIT|NOFRAME,$0 + // O=-288; for R in R{14..31}; do echo "\tMOVD\t$O(R1), $R"|sed s/R30/g/; ((O+=8)); done; for F in F{14..31}; do echo "\tFMOVD\t$O(R1), $F"; ((O+=8)); done + MOVD -288(R1), R14 + MOVD -280(R1), R15 + MOVD -272(R1), R16 + MOVD -264(R1), R17 + MOVD -256(R1), R18 + MOVD -248(R1), R19 + MOVD -240(R1), R20 + MOVD -232(R1), R21 + MOVD -224(R1), R22 + MOVD -216(R1), R23 + MOVD -208(R1), R24 + MOVD -200(R1), R25 + MOVD -192(R1), R26 + MOVD -184(R1), R27 + MOVD -176(R1), R28 + MOVD -168(R1), R29 + MOVD -160(R1), g + MOVD -152(R1), R31 + FMOVD -144(R1), F14 + FMOVD -136(R1), F15 + FMOVD -128(R1), F16 + FMOVD -120(R1), F17 + FMOVD -112(R1), F18 + FMOVD -104(R1), F19 + FMOVD -96(R1), F20 + FMOVD -88(R1), F21 + FMOVD -80(R1), F22 + FMOVD -72(R1), F23 + FMOVD -64(R1), F24 + FMOVD -56(R1), F25 + FMOVD -48(R1), F26 + FMOVD -40(R1), F27 + FMOVD -32(R1), F28 + FMOVD -24(R1), F29 + FMOVD -16(R1), F30 + FMOVD -8(R1), F31 + + RET diff --git a/src/runtime/cgo/asm_riscv64.s b/src/runtime/cgo/asm_riscv64.s new file mode 100644 index 0000000..45151bf --- /dev/null +++ b/src/runtime/cgo/asm_riscv64.s @@ -0,0 +1,78 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + /* + * Push arguments for fn (X10, X11, X13), along with all callee-save + * registers. Note that at procedure entry the first argument is at + * 8(X2). + */ + ADD $(-8*29), X2 + MOV X10, (8*1)(X2) // fn unsafe.Pointer + MOV X11, (8*2)(X2) // a unsafe.Pointer + MOV X13, (8*3)(X2) // ctxt uintptr + MOV X8, (8*4)(X2) + MOV X9, (8*5)(X2) + MOV X18, (8*6)(X2) + MOV X19, (8*7)(X2) + MOV X20, (8*8)(X2) + MOV X21, (8*9)(X2) + MOV X22, (8*10)(X2) + MOV X23, (8*11)(X2) + MOV X24, (8*12)(X2) + MOV X25, (8*13)(X2) + MOV X26, (8*14)(X2) + MOV g, (8*15)(X2) + MOV X1, (8*16)(X2) + MOVD F8, (8*17)(X2) + MOVD F9, (8*18)(X2) + MOVD F18, (8*19)(X2) + MOVD F19, (8*20)(X2) + MOVD F20, (8*21)(X2) + MOVD F21, (8*22)(X2) + MOVD F22, (8*23)(X2) + MOVD F23, (8*24)(X2) + MOVD F24, (8*25)(X2) + MOVD F25, (8*26)(X2) + MOVD F26, (8*27)(X2) + MOVD F27, (8*28)(X2) + + // Initialize Go ABI environment + CALL runtime·load_g(SB) + CALL runtime·cgocallback(SB) + + MOV (8*4)(X2), X8 + MOV (8*5)(X2), X9 + MOV (8*6)(X2), X18 + MOV (8*7)(X2), X19 + MOV (8*8)(X2), X20 + MOV (8*9)(X2), X21 + MOV (8*10)(X2), X22 + MOV (8*11)(X2), X23 + MOV (8*12)(X2), X24 + MOV (8*13)(X2), X25 + MOV (8*14)(X2), X26 + MOV (8*15)(X2), g + MOV (8*16)(X2), X1 + MOVD (8*17)(X2), F8 + MOVD (8*18)(X2), F9 + MOVD (8*19)(X2), F18 + MOVD (8*20)(X2), F19 + MOVD (8*21)(X2), F20 + MOVD (8*22)(X2), F21 + MOVD (8*23)(X2), F22 + MOVD (8*24)(X2), F23 + MOVD (8*25)(X2), F24 + MOVD (8*26)(X2), F25 + MOVD (8*27)(X2), F26 + MOVD (8*28)(X2), F27 + ADD $(8*29), X2 + + RET diff --git a/src/runtime/cgo/asm_s390x.s b/src/runtime/cgo/asm_s390x.s new file mode 100644 index 0000000..8bf16e7 --- /dev/null +++ b/src/runtime/cgo/asm_s390x.s @@ -0,0 +1,55 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 + // Start with standard C stack frame layout and linkage. + + // Save R6-R15 in the register save area of the calling function. + STMG R6, R15, 48(R15) + + // Allocate 96 bytes on the stack. + MOVD $-96(R15), R15 + + // Save F8-F15 in our stack frame. + FMOVD F8, 32(R15) + FMOVD F9, 40(R15) + FMOVD F10, 48(R15) + FMOVD F11, 56(R15) + FMOVD F12, 64(R15) + FMOVD F13, 72(R15) + FMOVD F14, 80(R15) + FMOVD F15, 88(R15) + + // Initialize Go ABI environment. + BL runtime·load_g(SB) + + MOVD R2, 8(R15) // fn unsafe.Pointer + MOVD R3, 16(R15) // a unsafe.Pointer + // Skip R4 = n uint32 + MOVD R5, 24(R15) // ctxt uintptr + BL runtime·cgocallback(SB) + + FMOVD 32(R15), F8 + FMOVD 40(R15), F9 + FMOVD 48(R15), F10 + FMOVD 56(R15), F11 + FMOVD 64(R15), F12 + FMOVD 72(R15), F13 + FMOVD 80(R15), F14 + FMOVD 88(R15), F15 + + // De-allocate stack frame. + MOVD $96(R15), R15 + + // Restore R6-R15. + LMG 48(R15), R6, R15 + + RET + diff --git a/src/runtime/cgo/asm_wasm.s b/src/runtime/cgo/asm_wasm.s new file mode 100644 index 0000000..cb140eb --- /dev/null +++ b/src/runtime/cgo/asm_wasm.s @@ -0,0 +1,8 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT crosscall2(SB), NOSPLIT, $0 + UNDEF diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go new file mode 100644 index 0000000..cd8b795 --- /dev/null +++ b/src/runtime/cgo/callbacks.go @@ -0,0 +1,106 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +import "unsafe" + +// These utility functions are available to be called from code +// compiled with gcc via crosscall2. + +// The declaration of crosscall2 is: +// void crosscall2(void (*fn)(void *), void *, int); +// +// We need to export the symbol crosscall2 in order to support +// callbacks from shared libraries. This applies regardless of +// linking mode. +// +// Compatibility note: SWIG uses crosscall2 in exactly one situation: +// to call _cgo_panic using the pattern shown below. We need to keep +// that pattern working. In particular, crosscall2 actually takes four +// arguments, but it works to call it with three arguments when +// calling _cgo_panic. +//go:cgo_export_static crosscall2 +//go:cgo_export_dynamic crosscall2 + +// Panic. The argument is converted into a Go string. + +// Call like this in code compiled with gcc: +// struct { const char *p; } a; +// a.p = /* string to pass to panic */; +// crosscall2(_cgo_panic, &a, sizeof a); +// /* The function call will not return. */ + +// TODO: We should export a regular C function to panic, change SWIG +// to use that instead of the above pattern, and then we can drop +// backwards-compatibility from crosscall2 and stop exporting it. + +//go:linkname _runtime_cgo_panic_internal runtime._cgo_panic_internal +func _runtime_cgo_panic_internal(p *byte) + +//go:linkname _cgo_panic _cgo_panic +//go:cgo_export_static _cgo_panic +//go:cgo_export_dynamic _cgo_panic +func _cgo_panic(a *struct{ cstr *byte }) { + _runtime_cgo_panic_internal(a.cstr) +} + +//go:cgo_import_static x_cgo_init +//go:linkname x_cgo_init x_cgo_init +//go:linkname _cgo_init _cgo_init +var x_cgo_init byte +var _cgo_init = &x_cgo_init + +//go:cgo_import_static x_cgo_thread_start +//go:linkname x_cgo_thread_start x_cgo_thread_start +//go:linkname _cgo_thread_start _cgo_thread_start +var x_cgo_thread_start byte +var _cgo_thread_start = &x_cgo_thread_start + +// Creates a new system thread without updating any Go state. +// +// This method is invoked during shared library loading to create a new OS +// thread to perform the runtime initialization. This method is similar to +// _cgo_sys_thread_start except that it doesn't update any Go state. + +//go:cgo_import_static x_cgo_sys_thread_create +//go:linkname x_cgo_sys_thread_create x_cgo_sys_thread_create +//go:linkname _cgo_sys_thread_create _cgo_sys_thread_create +var x_cgo_sys_thread_create byte +var _cgo_sys_thread_create = &x_cgo_sys_thread_create + +// Notifies that the runtime has been initialized. +// +// We currently block at every CGO entry point (via _cgo_wait_runtime_init_done) +// to ensure that the runtime has been initialized before the CGO call is +// executed. This is necessary for shared libraries where we kickoff runtime +// initialization in a separate thread and return without waiting for this +// thread to complete the init. + +//go:cgo_import_static x_cgo_notify_runtime_init_done +//go:linkname x_cgo_notify_runtime_init_done x_cgo_notify_runtime_init_done +//go:linkname _cgo_notify_runtime_init_done _cgo_notify_runtime_init_done +var x_cgo_notify_runtime_init_done byte +var _cgo_notify_runtime_init_done = &x_cgo_notify_runtime_init_done + +// Sets the traceback context function. See runtime.SetCgoTraceback. + +//go:cgo_import_static x_cgo_set_context_function +//go:linkname x_cgo_set_context_function x_cgo_set_context_function +//go:linkname _cgo_set_context_function _cgo_set_context_function +var x_cgo_set_context_function byte +var _cgo_set_context_function = &x_cgo_set_context_function + +// Calls a libc function to execute background work injected via libc +// interceptors, such as processing pending signals under the thread +// sanitizer. +// +// Left as a nil pointer if no libc interceptors are expected. + +//go:cgo_import_static _cgo_yield +//go:linkname _cgo_yield _cgo_yield +var _cgo_yield unsafe.Pointer + +//go:cgo_export_static _cgo_topofstack +//go:cgo_export_dynamic _cgo_topofstack diff --git a/src/runtime/cgo/callbacks_aix.go b/src/runtime/cgo/callbacks_aix.go new file mode 100644 index 0000000..f4b6fe2 --- /dev/null +++ b/src/runtime/cgo/callbacks_aix.go @@ -0,0 +1,11 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +// These functions must be exported in order to perform +// longcall on cgo programs (cf gcc_aix_ppc64.c). +//go:cgo_export_static __cgo_topofstack +//go:cgo_export_static runtime.rt0_go +//go:cgo_export_static _rt0_ppc64_aix_lib diff --git a/src/runtime/cgo/callbacks_traceback.go b/src/runtime/cgo/callbacks_traceback.go new file mode 100644 index 0000000..dae31a8 --- /dev/null +++ b/src/runtime/cgo/callbacks_traceback.go @@ -0,0 +1,17 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build darwin || linux + +package cgo + +import _ "unsafe" // for go:linkname + +// Calls the traceback function passed to SetCgoTraceback. + +//go:cgo_import_static x_cgo_callers +//go:linkname x_cgo_callers x_cgo_callers +//go:linkname _cgo_callers _cgo_callers +var x_cgo_callers byte +var _cgo_callers = &x_cgo_callers diff --git a/src/runtime/cgo/cgo.go b/src/runtime/cgo/cgo.go new file mode 100644 index 0000000..4d2caf6 --- /dev/null +++ b/src/runtime/cgo/cgo.go @@ -0,0 +1,34 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package cgo contains runtime support for code generated +by the cgo tool. See the documentation for the cgo command +for details on using cgo. +*/ +package cgo + +/* + +#cgo darwin,!arm64 LDFLAGS: -lpthread +#cgo darwin,arm64 LDFLAGS: -framework CoreFoundation +#cgo dragonfly LDFLAGS: -lpthread +#cgo freebsd LDFLAGS: -lpthread +#cgo android LDFLAGS: -llog +#cgo !android,linux LDFLAGS: -lpthread +#cgo netbsd LDFLAGS: -lpthread +#cgo openbsd LDFLAGS: -lpthread +#cgo aix LDFLAGS: -Wl,-berok +#cgo solaris LDFLAGS: -lxnet +#cgo illumos LDFLAGS: -lsocket + +// Issue 35247. +#cgo darwin CFLAGS: -Wno-nullability-completeness + +#cgo CFLAGS: -Wall -Werror + +#cgo solaris CPPFLAGS: -D_POSIX_PTHREAD_SEMANTICS + +*/ +import "C" diff --git a/src/runtime/cgo/dragonfly.go b/src/runtime/cgo/dragonfly.go new file mode 100644 index 0000000..36d70e3 --- /dev/null +++ b/src/runtime/cgo/dragonfly.go @@ -0,0 +1,19 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build dragonfly + +package cgo + +import _ "unsafe" // for go:linkname + +// Supply environ and __progname, because we don't +// link against the standard DragonFly crt0.o and the +// libc dynamic library needs them. + +//go:linkname _environ environ +//go:linkname _progname __progname + +var _environ uintptr +var _progname uintptr diff --git a/src/runtime/cgo/freebsd.go b/src/runtime/cgo/freebsd.go new file mode 100644 index 0000000..2d9f624 --- /dev/null +++ b/src/runtime/cgo/freebsd.go @@ -0,0 +1,22 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build freebsd + +package cgo + +import _ "unsafe" // for go:linkname + +// Supply environ and __progname, because we don't +// link against the standard FreeBSD crt0.o and the +// libc dynamic library needs them. + +//go:linkname _environ environ +//go:linkname _progname __progname + +//go:cgo_export_dynamic environ +//go:cgo_export_dynamic __progname + +var _environ uintptr +var _progname uintptr diff --git a/src/runtime/cgo/gcc_386.S b/src/runtime/cgo/gcc_386.S new file mode 100644 index 0000000..ff55b2c --- /dev/null +++ b/src/runtime/cgo/gcc_386.S @@ -0,0 +1,40 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Apple still insists on underscore prefixes for C function names. + */ +#if defined(__APPLE__) || defined(_WIN32) +#define EXT(s) _##s +#else +#define EXT(s) s +#endif + +/* + * void crosscall_386(void (*fn)(void)) + * + * Calling into the 8c tool chain, where all registers are caller save. + * Called from standard x86 ABI, where %ebp, %ebx, %esi, + * and %edi are callee-save, so they must be saved explicitly. + */ +.globl EXT(crosscall_386) +EXT(crosscall_386): + pushl %ebp + movl %esp, %ebp + pushl %ebx + pushl %esi + pushl %edi + + movl 8(%ebp), %eax /* fn */ + call *%eax + + popl %edi + popl %esi + popl %ebx + popl %ebp + ret + +#ifdef __ELF__ +.section .note.GNU-stack,"",@progbits +#endif diff --git a/src/runtime/cgo/gcc_aix_ppc64.S b/src/runtime/cgo/gcc_aix_ppc64.S new file mode 100644 index 0000000..a00fae2 --- /dev/null +++ b/src/runtime/cgo/gcc_aix_ppc64.S @@ -0,0 +1,133 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 +// +build aix + +/* + * void crosscall_ppc64(void (*fn)(void), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard ppc64 C ABI, where r2, r14-r31, f14-f31 are + * callee-save, so they must be saved explicitly. + * AIX has a special assembly syntax and keywords that can be mixed with + * Linux assembly. + */ + .toc + .csect .text[PR] + .globl crosscall_ppc64 + .globl .crosscall_ppc64 + .csect crosscall_ppc64[DS] +crosscall_ppc64: + .llong .crosscall_ppc64, TOC[tc0], 0 + .csect .text[PR] +.crosscall_ppc64: + // Start with standard C stack frame layout and linkage + mflr 0 + std 0, 16(1) // Save LR in caller's frame + std 2, 40(1) // Save TOC in caller's frame + bl saveregs + stdu 1, -296(1) + + // Set up Go ABI constant registers + // Must match _cgo_reginit in runtime package. + xor 0, 0, 0 + + // Restore g pointer (r30 in Go ABI, which may have been clobbered by C) + mr 30, 4 + + // Call fn + mr 12, 3 + mtctr 12 + bctrl + + addi 1, 1, 296 + bl restoreregs + ld 2, 40(1) + ld 0, 16(1) + mtlr 0 + blr + +saveregs: + // Save callee-save registers + // O=-288; for R in {14..31}; do echo "\tstd\t$R, $O(1)"; ((O+=8)); done; for F in f{14..31}; do echo "\tstfd\t$F, $O(1)"; ((O+=8)); done + std 14, -288(1) + std 15, -280(1) + std 16, -272(1) + std 17, -264(1) + std 18, -256(1) + std 19, -248(1) + std 20, -240(1) + std 21, -232(1) + std 22, -224(1) + std 23, -216(1) + std 24, -208(1) + std 25, -200(1) + std 26, -192(1) + std 27, -184(1) + std 28, -176(1) + std 29, -168(1) + std 30, -160(1) + std 31, -152(1) + stfd 14, -144(1) + stfd 15, -136(1) + stfd 16, -128(1) + stfd 17, -120(1) + stfd 18, -112(1) + stfd 19, -104(1) + stfd 20, -96(1) + stfd 21, -88(1) + stfd 22, -80(1) + stfd 23, -72(1) + stfd 24, -64(1) + stfd 25, -56(1) + stfd 26, -48(1) + stfd 27, -40(1) + stfd 28, -32(1) + stfd 29, -24(1) + stfd 30, -16(1) + stfd 31, -8(1) + + blr + +restoreregs: + // O=-288; for R in {14..31}; do echo "\tld\t$R, $O(1)"; ((O+=8)); done; for F in {14..31}; do echo "\tlfd\t$F, $O(1)"; ((O+=8)); done + ld 14, -288(1) + ld 15, -280(1) + ld 16, -272(1) + ld 17, -264(1) + ld 18, -256(1) + ld 19, -248(1) + ld 20, -240(1) + ld 21, -232(1) + ld 22, -224(1) + ld 23, -216(1) + ld 24, -208(1) + ld 25, -200(1) + ld 26, -192(1) + ld 27, -184(1) + ld 28, -176(1) + ld 29, -168(1) + ld 30, -160(1) + ld 31, -152(1) + lfd 14, -144(1) + lfd 15, -136(1) + lfd 16, -128(1) + lfd 17, -120(1) + lfd 18, -112(1) + lfd 19, -104(1) + lfd 20, -96(1) + lfd 21, -88(1) + lfd 22, -80(1) + lfd 23, -72(1) + lfd 24, -64(1) + lfd 25, -56(1) + lfd 26, -48(1) + lfd 27, -40(1) + lfd 28, -32(1) + lfd 29, -24(1) + lfd 30, -16(1) + lfd 31, -8(1) + + blr diff --git a/src/runtime/cgo/gcc_aix_ppc64.c b/src/runtime/cgo/gcc_aix_ppc64.c new file mode 100644 index 0000000..f4f50b8 --- /dev/null +++ b/src/runtime/cgo/gcc_aix_ppc64.c @@ -0,0 +1,38 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix +// +build ppc64 ppc64le + +/* + * On AIX, call to _cgo_topofstack and Go main are forced to be a longcall. + * Without it, ld might add trampolines in the middle of .text section + * to reach these functions which are normally declared in runtime package. + */ +extern int __attribute__((longcall)) __cgo_topofstack(void); +extern int __attribute__((longcall)) runtime_rt0_go(int argc, char **argv); +extern void __attribute__((longcall)) _rt0_ppc64_aix_lib(void); + +int _cgo_topofstack(void) { + return __cgo_topofstack(); +} + +int main(int argc, char **argv) { + return runtime_rt0_go(argc, argv); +} + +static void libinit(void) __attribute__ ((constructor)); + +/* + * libinit aims to replace .init_array section which isn't available on aix. + * Using __attribute__ ((constructor)) let gcc handles this instead of + * adding special code in cmd/link. + * However, it will be called for every Go programs which has cgo. + * Inside _rt0_ppc64_aix_lib(), runtime.isarchive is checked in order + * to know if this program is a c-archive or a simple cgo program. + * If it's not set, _rt0_ppc64_ax_lib() returns directly. + */ +static void libinit() { + _rt0_ppc64_aix_lib(); +} diff --git a/src/runtime/cgo/gcc_amd64.S b/src/runtime/cgo/gcc_amd64.S new file mode 100644 index 0000000..46699d1 --- /dev/null +++ b/src/runtime/cgo/gcc_amd64.S @@ -0,0 +1,53 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Apple still insists on underscore prefixes for C function names. + */ +#if defined(__APPLE__) +#define EXT(s) _##s +#else +#define EXT(s) s +#endif + +/* + * void crosscall_amd64(void (*fn)(void), void (*setg_gcc)(void*), void *g) + * + * Calling into the 6c tool chain, where all registers are caller save. + * Called from standard x86-64 ABI, where %rbx, %rbp, %r12-%r15 + * are callee-save so they must be saved explicitly. + * The standard x86-64 ABI passes the three arguments m, g, fn + * in %rdi, %rsi, %rdx. + */ +.globl EXT(crosscall_amd64) +EXT(crosscall_amd64): + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + +#if defined(_WIN64) + movq %r8, %rdi /* arg of setg_gcc */ + call *%rdx /* setg_gcc */ + call *%rcx /* fn */ +#else + movq %rdi, %rbx + movq %rdx, %rdi /* arg of setg_gcc */ + call *%rsi /* setg_gcc */ + call *%rbx /* fn */ +#endif + + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + ret + +#ifdef __ELF__ +.section .note.GNU-stack,"",@progbits +#endif diff --git a/src/runtime/cgo/gcc_android.c b/src/runtime/cgo/gcc_android.c new file mode 100644 index 0000000..7ea2135 --- /dev/null +++ b/src/runtime/cgo/gcc_android.c @@ -0,0 +1,90 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <stdarg.h> +#include <android/log.h> +#include <pthread.h> +#include <dlfcn.h> +#include "libcgo.h" + +void +fatalf(const char* format, ...) +{ + va_list ap; + + // Write to both stderr and logcat. + // + // When running from an .apk, /dev/stderr and /dev/stdout + // redirect to /dev/null. And when running a test binary + // via adb shell, it's easy to miss logcat. + + fprintf(stderr, "runtime/cgo: "); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fprintf(stderr, "\n"); + + va_start(ap, format); + __android_log_vprint(ANDROID_LOG_FATAL, "runtime/cgo", format, ap); + va_end(ap); + + abort(); +} + +// Truncated to a different magic value on 32-bit; that's ok. +#define magic1 (0x23581321345589ULL) + +// From https://android.googlesource.com/platform/bionic/+/refs/heads/android10-tests-release/libc/private/bionic_asm_tls.h#69. +#define TLS_SLOT_APP 2 + +// inittls allocates a thread-local storage slot for g. +// +// It finds the first available slot using pthread_key_create and uses +// it as the offset value for runtime.tls_g. +static void +inittls(void **tlsg, void **tlsbase) +{ + pthread_key_t k; + int i, err; + void *handle, *get_ver, *off; + + // Check for Android Q where we can use the free TLS_SLOT_APP slot. + handle = dlopen("libc.so", RTLD_LAZY); + if (handle == NULL) { + fatalf("inittls: failed to dlopen main program"); + return; + } + // android_get_device_api_level is introduced in Android Q, so its mere presence + // is enough. + get_ver = dlsym(handle, "android_get_device_api_level"); + dlclose(handle); + if (get_ver != NULL) { + off = (void *)(TLS_SLOT_APP*sizeof(void *)); + // tlsg is initialized to Q's free TLS slot. Verify it while we're here. + if (*tlsg != off) { + fatalf("tlsg offset wrong, got %ld want %ld\n", *tlsg, off); + } + return; + } + + err = pthread_key_create(&k, nil); + if(err != 0) { + fatalf("pthread_key_create failed: %d", err); + } + pthread_setspecific(k, (void*)magic1); + // If thread local slots are laid out as we expect, our magic word will + // be located at some low offset from tlsbase. However, just in case something went + // wrong, the search is limited to sensible offsets. PTHREAD_KEYS_MAX was the + // original limit, but issue 19472 made a higher limit necessary. + for (i=0; i<384; i++) { + if (*(tlsbase+i) == (void*)magic1) { + *tlsg = (void*)(i*sizeof(void *)); + pthread_setspecific(k, 0); + return; + } + } + fatalf("inittls: could not find pthread key"); +} + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase) = inittls; diff --git a/src/runtime/cgo/gcc_arm.S b/src/runtime/cgo/gcc_arm.S new file mode 100644 index 0000000..fe1c48b --- /dev/null +++ b/src/runtime/cgo/gcc_arm.S @@ -0,0 +1,42 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Apple still insists on underscore prefixes for C function names. + */ +#if defined(__APPLE__) +#define EXT(s) _##s +#else +#define EXT(s) s +#endif + +// Apple's ld64 wants 4-byte alignment for ARM code sections. +// .align in both Apple as and GNU as treat n as aligning to 2**n bytes. +.align 2 + +/* + * void crosscall_arm1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the 5c tool chain, where all registers are caller save. + * Called from standard ARM EABI, where r4-r11 are callee-save, so they + * must be saved explicitly. + */ +.globl EXT(crosscall_arm1) +EXT(crosscall_arm1): + push {r4, r5, r6, r7, r8, r9, r10, r11, ip, lr} + mov r4, r0 + mov r5, r1 + mov r0, r2 + + // Because the assembler might target an earlier revision of the ISA + // by default, we encode BLX as a .word. + .word 0xe12fff35 // blx r5 // setg(g) + .word 0xe12fff34 // blx r4 // fn() + + pop {r4, r5, r6, r7, r8, r9, r10, r11, ip, pc} + + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_arm64.S b/src/runtime/cgo/gcc_arm64.S new file mode 100644 index 0000000..9154d2a --- /dev/null +++ b/src/runtime/cgo/gcc_arm64.S @@ -0,0 +1,82 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Apple still insists on underscore prefixes for C function names. + */ +#if defined(__APPLE__) +#define EXT(s) _##s +#else +#define EXT(s) s +#endif + +// Apple's ld64 wants 4-byte alignment for ARM code sections. +// .align in both Apple as and GNU as treat n as aligning to 2**n bytes. +.align 2 + +/* + * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard ARM EABI, where x19-x29 are callee-save, so they + * must be saved explicitly, along with x30 (LR). + */ +.globl EXT(crosscall1) +EXT(crosscall1): + .cfi_startproc + stp x29, x30, [sp, #-96]! + .cfi_def_cfa_offset 96 + .cfi_offset 29, -96 + .cfi_offset 30, -88 + mov x29, sp + .cfi_def_cfa_register 29 + stp x19, x20, [sp, #80] + .cfi_offset 19, -16 + .cfi_offset 20, -8 + stp x21, x22, [sp, #64] + .cfi_offset 21, -32 + .cfi_offset 22, -24 + stp x23, x24, [sp, #48] + .cfi_offset 23, -48 + .cfi_offset 24, -40 + stp x25, x26, [sp, #32] + .cfi_offset 25, -64 + .cfi_offset 26, -56 + stp x27, x28, [sp, #16] + .cfi_offset 27, -80 + .cfi_offset 28, -72 + + mov x19, x0 + mov x20, x1 + mov x0, x2 + + blr x20 + blr x19 + + ldp x27, x28, [sp, #16] + .cfi_restore 27 + .cfi_restore 28 + ldp x25, x26, [sp, #32] + .cfi_restore 25 + .cfi_restore 26 + ldp x23, x24, [sp, #48] + .cfi_restore 23 + .cfi_restore 24 + ldp x21, x22, [sp, #64] + .cfi_restore 21 + .cfi_restore 22 + ldp x19, x20, [sp, #80] + .cfi_restore 19 + .cfi_restore 20 + ldp x29, x30, [sp], #96 + .cfi_restore 29 + .cfi_restore 30 + .cfi_def_cfa 31, 0 + ret + .cfi_endproc + + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_context.c b/src/runtime/cgo/gcc_context.c new file mode 100644 index 0000000..5fc0abb --- /dev/null +++ b/src/runtime/cgo/gcc_context.c @@ -0,0 +1,21 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris windows + +#include "libcgo.h" + +// Releases the cgo traceback context. +void _cgo_release_context(uintptr_t ctxt) { + void (*pfn)(struct context_arg*); + + pfn = _cgo_get_context_function(); + if (ctxt != 0 && pfn != nil) { + struct context_arg arg; + + arg.Context = ctxt; + (*pfn)(&arg); + } +} diff --git a/src/runtime/cgo/gcc_darwin_amd64.c b/src/runtime/cgo/gcc_darwin_amd64.c new file mode 100644 index 0000000..d5b7fd8 --- /dev/null +++ b/src/runtime/cgo/gcc_darwin_amd64.c @@ -0,0 +1,65 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <string.h> /* for strerror */ +#include <pthread.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_darwin_arm64.c b/src/runtime/cgo/gcc_darwin_arm64.c new file mode 100644 index 0000000..a5f07f1 --- /dev/null +++ b/src/runtime/cgo/gcc_darwin_arm64.c @@ -0,0 +1,145 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <limits.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> /* for strerror */ +#include <sys/param.h> +#include <unistd.h> +#include <stdlib.h> + +#include "libcgo.h" +#include "libcgo_unix.h" + +#include <TargetConditionals.h> + +#if TARGET_OS_IPHONE +#include <CoreFoundation/CFBundle.h> +#include <CoreFoundation/CFString.h> +#endif + +static void *threadentry(void*); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + //fprintf(stderr, "runtime/cgo: _cgo_sys_thread_start: fn=%p, g=%p\n", ts->fn, ts->g); // debug + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + +#if TARGET_OS_IPHONE + darwin_arm_init_thread_exception_port(); +#endif + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +#if TARGET_OS_IPHONE + +// init_working_dir sets the current working directory to the app root. +// By default ios/arm64 processes start in "/". +static void +init_working_dir() +{ + CFBundleRef bundle = CFBundleGetMainBundle(); + if (bundle == NULL) { + fprintf(stderr, "runtime/cgo: no main bundle\n"); + return; + } + CFURLRef url_ref = CFBundleCopyResourceURL(bundle, CFSTR("Info"), CFSTR("plist"), NULL); + if (url_ref == NULL) { + // No Info.plist found. It can happen on Corellium virtual devices. + return; + } + CFStringRef url_str_ref = CFURLGetString(url_ref); + char buf[MAXPATHLEN]; + Boolean res = CFStringGetCString(url_str_ref, buf, sizeof(buf), kCFStringEncodingUTF8); + CFRelease(url_ref); + if (!res) { + fprintf(stderr, "runtime/cgo: cannot get URL string\n"); + return; + } + + // url is of the form "file:///path/to/Info.plist". + // strip it down to the working directory "/path/to". + int url_len = strlen(buf); + if (url_len < sizeof("file://")+sizeof("/Info.plist")) { + fprintf(stderr, "runtime/cgo: bad URL: %s\n", buf); + return; + } + buf[url_len-sizeof("/Info.plist")+1] = 0; + char *dir = &buf[0] + sizeof("file://")-1; + + if (chdir(dir) != 0) { + fprintf(stderr, "runtime/cgo: chdir(%s) failed\n", dir); + } + + // The test harness in go_ios_exec passes the relative working directory + // in the GoExecWrapperWorkingDirectory property of the app bundle. + CFStringRef wd_ref = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("GoExecWrapperWorkingDirectory")); + if (wd_ref != NULL) { + if (!CFStringGetCString(wd_ref, buf, sizeof(buf), kCFStringEncodingUTF8)) { + fprintf(stderr, "runtime/cgo: cannot get GoExecWrapperWorkingDirectory string\n"); + return; + } + if (chdir(buf) != 0) { + fprintf(stderr, "runtime/cgo: chdir(%s) failed\n", buf); + } + } +} + +#endif // TARGET_OS_IPHONE + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + //fprintf(stderr, "x_cgo_init = %p\n", &x_cgo_init); // aid debugging in presence of ASLR + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + +#if TARGET_OS_IPHONE + darwin_arm_init_mach_exception_handler(); + darwin_arm_init_thread_exception_port(); + init_working_dir(); +#endif +} diff --git a/src/runtime/cgo/gcc_dragonfly_amd64.c b/src/runtime/cgo/gcc_dragonfly_amd64.c new file mode 100644 index 0000000..0003414 --- /dev/null +++ b/src/runtime/cgo/gcc_dragonfly_amd64.c @@ -0,0 +1,66 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <sys/signalvar.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + SIGFILLSET(ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_fatalf.c b/src/runtime/cgo/gcc_fatalf.c new file mode 100644 index 0000000..597e750 --- /dev/null +++ b/src/runtime/cgo/gcc_fatalf.c @@ -0,0 +1,23 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix !android,linux freebsd + +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include "libcgo.h" + +void +fatalf(const char* format, ...) +{ + va_list ap; + + fprintf(stderr, "runtime/cgo: "); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fprintf(stderr, "\n"); + abort(); +} diff --git a/src/runtime/cgo/gcc_freebsd_386.c b/src/runtime/cgo/gcc_freebsd_386.c new file mode 100644 index 0000000..9097a2a --- /dev/null +++ b/src/runtime/cgo/gcc_freebsd_386.c @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <sys/signalvar.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + SIGFILLSET(ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + /* + * Set specific keys. + */ + setg_gcc((void*)ts.g); + + crosscall_386(ts.fn); + return nil; +} diff --git a/src/runtime/cgo/gcc_freebsd_amd64.c b/src/runtime/cgo/gcc_freebsd_amd64.c new file mode 100644 index 0000000..6071ec3 --- /dev/null +++ b/src/runtime/cgo/gcc_freebsd_amd64.c @@ -0,0 +1,74 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <errno.h> +#include <sys/signalvar.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t *attr; + size_t size; + + // Deal with memory sanitizer/clang interaction. + // See gcc_linux_amd64.c for details. + setg_gcc = setg; + attr = (pthread_attr_t*)malloc(sizeof *attr); + if (attr == NULL) { + fatalf("malloc failed: %s", strerror(errno)); + } + pthread_attr_init(attr); + pthread_attr_getstacksize(attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(attr); + free(attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + SIGFILLSET(ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + _cgo_tsan_acquire(); + free(v); + _cgo_tsan_release(); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_freebsd_arm.c b/src/runtime/cgo/gcc_freebsd_arm.c new file mode 100644 index 0000000..74f2e0e --- /dev/null +++ b/src/runtime/cgo/gcc_freebsd_arm.c @@ -0,0 +1,83 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <machine/sysarch.h> +#include <sys/signalvar.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +#ifdef ARM_TP_ADDRESS +// ARM_TP_ADDRESS is (ARM_VECTORS_HIGH + 0x1000) or 0xffff1000 +// and is known to runtime.read_tls_fallback. Verify it with +// cpp. +#if ARM_TP_ADDRESS != 0xffff1000 +#error Wrong ARM_TP_ADDRESS! +#endif +#endif + +static void *threadentry(void*); + +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + SIGFILLSET(ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall_arm1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_arm1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_freebsd_arm64.c b/src/runtime/cgo/gcc_freebsd_arm64.c new file mode 100644 index 0000000..dd8f888 --- /dev/null +++ b/src/runtime/cgo/gcc_freebsd_arm64.c @@ -0,0 +1,68 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <errno.h> +#include <sys/signalvar.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + SIGFILLSET(ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_freebsd_sigaction.c b/src/runtime/cgo/gcc_freebsd_sigaction.c new file mode 100644 index 0000000..98b122d --- /dev/null +++ b/src/runtime/cgo/gcc_freebsd_sigaction.c @@ -0,0 +1,80 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd,amd64 + +#include <errno.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include <signal.h> + +#include "libcgo.h" + +// go_sigaction_t is a C version of the sigactiont struct from +// os_freebsd.go. This definition — and its conversion to and from struct +// sigaction — are specific to freebsd/amd64. +typedef struct { + uint32_t __bits[_SIG_WORDS]; +} go_sigset_t; +typedef struct { + uintptr_t handler; + int32_t flags; + go_sigset_t mask; +} go_sigaction_t; + +int32_t +x_cgo_sigaction(intptr_t signum, const go_sigaction_t *goact, go_sigaction_t *oldgoact) { + int32_t ret; + struct sigaction act; + struct sigaction oldact; + size_t i; + + _cgo_tsan_acquire(); + + memset(&act, 0, sizeof act); + memset(&oldact, 0, sizeof oldact); + + if (goact) { + if (goact->flags & SA_SIGINFO) { + act.sa_sigaction = (void(*)(int, siginfo_t*, void*))(goact->handler); + } else { + act.sa_handler = (void(*)(int))(goact->handler); + } + sigemptyset(&act.sa_mask); + for (i = 0; i < 8 * sizeof(goact->mask); i++) { + if (goact->mask.__bits[i/32] & ((uint32_t)(1)<<(i&31))) { + sigaddset(&act.sa_mask, i+1); + } + } + act.sa_flags = goact->flags; + } + + ret = sigaction(signum, goact ? &act : NULL, oldgoact ? &oldact : NULL); + if (ret == -1) { + // runtime.sigaction expects _cgo_sigaction to return errno on error. + _cgo_tsan_release(); + return errno; + } + + if (oldgoact) { + if (oldact.sa_flags & SA_SIGINFO) { + oldgoact->handler = (uintptr_t)(oldact.sa_sigaction); + } else { + oldgoact->handler = (uintptr_t)(oldact.sa_handler); + } + for (i = 0 ; i < _SIG_WORDS; i++) { + oldgoact->mask.__bits[i] = 0; + } + for (i = 0; i < 8 * sizeof(oldgoact->mask); i++) { + if (sigismember(&oldact.sa_mask, i+1) == 1) { + oldgoact->mask.__bits[i/32] |= (uint32_t)(1)<<(i&31); + } + } + oldgoact->flags = oldact.sa_flags; + } + + _cgo_tsan_release(); + return ret; +} diff --git a/src/runtime/cgo/gcc_libinit.c b/src/runtime/cgo/gcc_libinit.c new file mode 100644 index 0000000..3304d95 --- /dev/null +++ b/src/runtime/cgo/gcc_libinit.c @@ -0,0 +1,113 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +#include <pthread.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> // strerror +#include <time.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static pthread_cond_t runtime_init_cond = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t runtime_init_mu = PTHREAD_MUTEX_INITIALIZER; +static int runtime_init_done; + +// The context function, used when tracing back C calls into Go. +static void (*cgo_context_function)(struct context_arg*); + +void +x_cgo_sys_thread_create(void* (*func)(void*), void* arg) { + pthread_t p; + int err = _cgo_try_pthread_create(&p, NULL, func, arg); + if (err != 0) { + fprintf(stderr, "pthread_create failed: %s", strerror(err)); + abort(); + } +} + +uintptr_t +_cgo_wait_runtime_init_done(void) { + void (*pfn)(struct context_arg*); + + pthread_mutex_lock(&runtime_init_mu); + while (runtime_init_done == 0) { + pthread_cond_wait(&runtime_init_cond, &runtime_init_mu); + } + + // TODO(iant): For the case of a new C thread calling into Go, such + // as when using -buildmode=c-archive, we know that Go runtime + // initialization is complete but we do not know that all Go init + // functions have been run. We should not fetch cgo_context_function + // until they have been, because that is where a call to + // SetCgoTraceback is likely to occur. We are going to wait for Go + // initialization to be complete anyhow, later, by waiting for + // main_init_done to be closed in cgocallbackg1. We should wait here + // instead. See also issue #15943. + pfn = cgo_context_function; + + pthread_mutex_unlock(&runtime_init_mu); + if (pfn != nil) { + struct context_arg arg; + + arg.Context = 0; + (*pfn)(&arg); + return arg.Context; + } + return 0; +} + +void +x_cgo_notify_runtime_init_done(void* dummy __attribute__ ((unused))) { + pthread_mutex_lock(&runtime_init_mu); + runtime_init_done = 1; + pthread_cond_broadcast(&runtime_init_cond); + pthread_mutex_unlock(&runtime_init_mu); +} + +// Sets the context function to call to record the traceback context +// when calling a Go function from C code. Called from runtime.SetCgoTraceback. +void x_cgo_set_context_function(void (*context)(struct context_arg*)) { + pthread_mutex_lock(&runtime_init_mu); + cgo_context_function = context; + pthread_mutex_unlock(&runtime_init_mu); +} + +// Gets the context function. +void (*(_cgo_get_context_function(void)))(struct context_arg*) { + void (*ret)(struct context_arg*); + + pthread_mutex_lock(&runtime_init_mu); + ret = cgo_context_function; + pthread_mutex_unlock(&runtime_init_mu); + return ret; +} + +// _cgo_try_pthread_create retries pthread_create if it fails with +// EAGAIN. +int +_cgo_try_pthread_create(pthread_t* thread, const pthread_attr_t* attr, void* (*pfn)(void*), void* arg) { + int tries; + int err; + struct timespec ts; + + for (tries = 0; tries < 20; tries++) { + err = pthread_create(thread, attr, pfn, arg); + if (err == 0) { + pthread_detach(*thread); + return 0; + } + if (err != EAGAIN) { + return err; + } + ts.tv_sec = 0; + ts.tv_nsec = (tries + 1) * 1000 * 1000; // Milliseconds. + nanosleep(&ts, nil); + } + return EAGAIN; +} diff --git a/src/runtime/cgo/gcc_libinit_windows.c b/src/runtime/cgo/gcc_libinit_windows.c new file mode 100644 index 0000000..ad50386 --- /dev/null +++ b/src/runtime/cgo/gcc_libinit_windows.c @@ -0,0 +1,125 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> + +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> + +#include "libcgo.h" + +static volatile LONG runtime_init_once_gate = 0; +static volatile LONG runtime_init_once_done = 0; + +static CRITICAL_SECTION runtime_init_cs; + +static HANDLE runtime_init_wait; +static int runtime_init_done; + +// Pre-initialize the runtime synchronization objects +void +_cgo_preinit_init() { + runtime_init_wait = CreateEvent(NULL, TRUE, FALSE, NULL); + if (runtime_init_wait == NULL) { + fprintf(stderr, "runtime: failed to create runtime initialization wait event.\n"); + abort(); + } + + InitializeCriticalSection(&runtime_init_cs); +} + +// Make sure that the preinit sequence has run. +void +_cgo_maybe_run_preinit() { + if (!InterlockedExchangeAdd(&runtime_init_once_done, 0)) { + if (InterlockedIncrement(&runtime_init_once_gate) == 1) { + _cgo_preinit_init(); + InterlockedIncrement(&runtime_init_once_done); + } else { + // Decrement to avoid overflow. + InterlockedDecrement(&runtime_init_once_gate); + while(!InterlockedExchangeAdd(&runtime_init_once_done, 0)) { + Sleep(0); + } + } + } +} + +void +x_cgo_sys_thread_create(void (*func)(void*), void* arg) { + uintptr_t thandle; + + thandle = _beginthread(func, 0, arg); + if(thandle == -1) { + fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); + abort(); + } +} + +int +_cgo_is_runtime_initialized() { + EnterCriticalSection(&runtime_init_cs); + int status = runtime_init_done; + LeaveCriticalSection(&runtime_init_cs); + return status; +} + +uintptr_t +_cgo_wait_runtime_init_done(void) { + void (*pfn)(struct context_arg*); + + _cgo_maybe_run_preinit(); + while (!_cgo_is_runtime_initialized()) { + WaitForSingleObject(runtime_init_wait, INFINITE); + } + pfn = _cgo_get_context_function(); + if (pfn != nil) { + struct context_arg arg; + + arg.Context = 0; + (*pfn)(&arg); + return arg.Context; + } + return 0; +} + +void +x_cgo_notify_runtime_init_done(void* dummy) { + _cgo_maybe_run_preinit(); + + EnterCriticalSection(&runtime_init_cs); + runtime_init_done = 1; + LeaveCriticalSection(&runtime_init_cs); + + if (!SetEvent(runtime_init_wait)) { + fprintf(stderr, "runtime: failed to signal runtime initialization complete.\n"); + abort(); + } +} + +// The context function, used when tracing back C calls into Go. +static void (*cgo_context_function)(struct context_arg*); + +// Sets the context function to call to record the traceback context +// when calling a Go function from C code. Called from runtime.SetCgoTraceback. +void x_cgo_set_context_function(void (*context)(struct context_arg*)) { + EnterCriticalSection(&runtime_init_cs); + cgo_context_function = context; + LeaveCriticalSection(&runtime_init_cs); +} + +// Gets the context function. +void (*(_cgo_get_context_function(void)))(struct context_arg*) { + void (*ret)(struct context_arg*); + + EnterCriticalSection(&runtime_init_cs); + ret = cgo_context_function; + LeaveCriticalSection(&runtime_init_cs); + return ret; +} diff --git a/src/runtime/cgo/gcc_linux_386.c b/src/runtime/cgo/gcc_linux_386.c new file mode 100644 index 0000000..70c942a --- /dev/null +++ b/src/runtime/cgo/gcc_linux_386.c @@ -0,0 +1,79 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); +static void (*setg_gcc)(void*); + +// This will be set in gcc_android.c for android-specific customization. +void (*x_cgo_inittls)(void **tlsg, void **tlsbase) __attribute__((common)); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + /* + * Set specific keys. + */ + setg_gcc((void*)ts.g); + + crosscall_386(ts.fn); + return nil; +} diff --git a/src/runtime/cgo/gcc_linux_amd64.c b/src/runtime/cgo/gcc_linux_amd64.c new file mode 100644 index 0000000..c25e7e7 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_amd64.c @@ -0,0 +1,94 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <errno.h> +#include <string.h> // strerror +#include <signal.h> +#include <stdlib.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +// This will be set in gcc_android.c for android-specific customization. +void (*x_cgo_inittls)(void **tlsg, void **tlsbase) __attribute__((common)); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t *attr; + size_t size; + + /* The memory sanitizer distributed with versions of clang + before 3.8 has a bug: if you call mmap before malloc, mmap + may return an address that is later overwritten by the msan + library. Avoid this problem by forcing a call to malloc + here, before we ever call malloc. + + This is only required for the memory sanitizer, so it's + unfortunate that we always run it. It should be possible + to remove this when we no longer care about versions of + clang before 3.8. The test for this is + misc/cgo/testsanitizers. + + GCC works hard to eliminate a seemingly unnecessary call to + malloc, so we actually use the memory we allocate. */ + + setg_gcc = setg; + attr = (pthread_attr_t*)malloc(sizeof *attr); + if (attr == NULL) { + fatalf("malloc failed: %s", strerror(errno)); + } + pthread_attr_init(attr); + pthread_attr_getstacksize(attr, &size); + g->stacklo = (uintptr)&size - size + 4096; + pthread_attr_destroy(attr); + free(attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + _cgo_tsan_acquire(); + free(v); + _cgo_tsan_release(); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_linux_arm.c b/src/runtime/cgo/gcc_linux_arm.c new file mode 100644 index 0000000..5bc0fee --- /dev/null +++ b/src/runtime/cgo/gcc_linux_arm.c @@ -0,0 +1,74 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase) __attribute__((common)); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall_arm1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_arm1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_arm64.c b/src/runtime/cgo/gcc_linux_arm64.c new file mode 100644 index 0000000..17ff274 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_arm64.c @@ -0,0 +1,96 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <errno.h> +#include <string.h> +#include <signal.h> +#include <stdlib.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase) __attribute__((common)); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t *attr; + size_t size; + + /* The memory sanitizer distributed with versions of clang + before 3.8 has a bug: if you call mmap before malloc, mmap + may return an address that is later overwritten by the msan + library. Avoid this problem by forcing a call to malloc + here, before we ever call malloc. + + This is only required for the memory sanitizer, so it's + unfortunate that we always run it. It should be possible + to remove this when we no longer care about versions of + clang before 3.8. The test for this is + misc/cgo/testsanitizers. + + GCC works hard to eliminate a seemingly unnecessary call to + malloc, so we actually use the memory we allocate. */ + + setg_gcc = setg; + attr = (pthread_attr_t*)malloc(sizeof *attr); + if (attr == NULL) { + fatalf("malloc failed: %s", strerror(errno)); + } + pthread_attr_init(attr); + pthread_attr_getstacksize(attr, &size); + g->stacklo = (uintptr)&size - size + 4096; + pthread_attr_destroy(attr); + free(attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_mips64x.c b/src/runtime/cgo/gcc_linux_mips64x.c new file mode 100644 index 0000000..42837b1 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_mips64x.c @@ -0,0 +1,78 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build linux +// +build mips64 mips64le + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_mipsx.c b/src/runtime/cgo/gcc_linux_mipsx.c new file mode 100644 index 0000000..a44ea30 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_mipsx.c @@ -0,0 +1,80 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build linux +// +build mips mipsle + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_ppc64x.S b/src/runtime/cgo/gcc_linux_ppc64x.S new file mode 100644 index 0000000..595eb38 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_ppc64x.S @@ -0,0 +1,138 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le +// +build linux + +/* + * Apple still insists on underscore prefixes for C function names. + */ +#if defined(__APPLE__) +#define EXT(s) _##s +#else +#define EXT(s) s +#endif + +/* + * void crosscall_ppc64(void (*fn)(void), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard ppc64 C ABI, where r2, r14-r31, f14-f31 are + * callee-save, so they must be saved explicitly. + */ +.globl EXT(crosscall_ppc64) +EXT(crosscall_ppc64): + // Start with standard C stack frame layout and linkage + mflr %r0 + std %r0, 16(%r1) // Save LR in caller's frame + std %r2, 24(%r1) // Save TOC in caller's frame + bl saveregs + stdu %r1, -296(%r1) + + // Set up Go ABI constant registers + bl _cgo_reginit + nop + + // Restore g pointer (r30 in Go ABI, which may have been clobbered by C) + mr %r30, %r4 + + // Call fn + mr %r12, %r3 + mtctr %r3 + bctrl + + addi %r1, %r1, 296 + bl restoreregs + ld %r2, 24(%r1) + ld %r0, 16(%r1) + mtlr %r0 + blr + +saveregs: + // Save callee-save registers + // O=-288; for R in %r{14..31}; do echo "\tstd\t$R, $O(%r1)"; ((O+=8)); done; for F in f{14..31}; do echo "\tstfd\t$F, $O(%r1)"; ((O+=8)); done + std %r14, -288(%r1) + std %r15, -280(%r1) + std %r16, -272(%r1) + std %r17, -264(%r1) + std %r18, -256(%r1) + std %r19, -248(%r1) + std %r20, -240(%r1) + std %r21, -232(%r1) + std %r22, -224(%r1) + std %r23, -216(%r1) + std %r24, -208(%r1) + std %r25, -200(%r1) + std %r26, -192(%r1) + std %r27, -184(%r1) + std %r28, -176(%r1) + std %r29, -168(%r1) + std %r30, -160(%r1) + std %r31, -152(%r1) + stfd %f14, -144(%r1) + stfd %f15, -136(%r1) + stfd %f16, -128(%r1) + stfd %f17, -120(%r1) + stfd %f18, -112(%r1) + stfd %f19, -104(%r1) + stfd %f20, -96(%r1) + stfd %f21, -88(%r1) + stfd %f22, -80(%r1) + stfd %f23, -72(%r1) + stfd %f24, -64(%r1) + stfd %f25, -56(%r1) + stfd %f26, -48(%r1) + stfd %f27, -40(%r1) + stfd %f28, -32(%r1) + stfd %f29, -24(%r1) + stfd %f30, -16(%r1) + stfd %f31, -8(%r1) + + blr + +restoreregs: + // O=-288; for R in %r{14..31}; do echo "\tld\t$R, $O(%r1)"; ((O+=8)); done; for F in %f{14..31}; do echo "\tlfd\t$F, $O(%r1)"; ((O+=8)); done + ld %r14, -288(%r1) + ld %r15, -280(%r1) + ld %r16, -272(%r1) + ld %r17, -264(%r1) + ld %r18, -256(%r1) + ld %r19, -248(%r1) + ld %r20, -240(%r1) + ld %r21, -232(%r1) + ld %r22, -224(%r1) + ld %r23, -216(%r1) + ld %r24, -208(%r1) + ld %r25, -200(%r1) + ld %r26, -192(%r1) + ld %r27, -184(%r1) + ld %r28, -176(%r1) + ld %r29, -168(%r1) + ld %r30, -160(%r1) + ld %r31, -152(%r1) + lfd %f14, -144(%r1) + lfd %f15, -136(%r1) + lfd %f16, -128(%r1) + lfd %f17, -120(%r1) + lfd %f18, -112(%r1) + lfd %f19, -104(%r1) + lfd %f20, -96(%r1) + lfd %f21, -88(%r1) + lfd %f22, -80(%r1) + lfd %f23, -72(%r1) + lfd %f24, -64(%r1) + lfd %f25, -56(%r1) + lfd %f26, -48(%r1) + lfd %f27, -40(%r1) + lfd %f28, -32(%r1) + lfd %f29, -24(%r1) + lfd %f30, -16(%r1) + lfd %f31, -8(%r1) + + blr + + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_linux_riscv64.c b/src/runtime/cgo/gcc_linux_riscv64.c new file mode 100644 index 0000000..22b76c2 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_riscv64.c @@ -0,0 +1,74 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +static void (*setg_gcc)(void*); + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Not sure why the memset is necessary here, + // but without it, we get a bogus stack size + // out of pthread_attr_getstacksize. C'est la Linux. + memset(&attr, 0, sizeof attr); + pthread_attr_init(&attr); + size = 0; + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); + + if (x_cgo_inittls) { + x_cgo_inittls(tlsg, tlsbase); + } +} diff --git a/src/runtime/cgo/gcc_linux_s390x.c b/src/runtime/cgo/gcc_linux_s390x.c new file mode 100644 index 0000000..bb60048 --- /dev/null +++ b/src/runtime/cgo/gcc_linux_s390x.c @@ -0,0 +1,69 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall_s390x(void (*fn)(void), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + // Save g for this thread in C TLS + setg_gcc((void*)ts.g); + + crosscall_s390x(ts.fn, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_mips64x.S b/src/runtime/cgo/gcc_mips64x.S new file mode 100644 index 0000000..908dd21 --- /dev/null +++ b/src/runtime/cgo/gcc_mips64x.S @@ -0,0 +1,87 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips64 mips64le + +/* + * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard MIPS N64 ABI, where $16-$23, $28, $30, and $f24-$f31 + * are callee-save, so they must be saved explicitly, along with $31 (LR). + */ +.globl crosscall1 +.set noat +crosscall1: +#ifndef __mips_soft_float + daddiu $29, $29, -160 +#else + daddiu $29, $29, -96 // For soft-float, no need to make room for FP registers +#endif + sd $31, 0($29) + sd $16, 8($29) + sd $17, 16($29) + sd $18, 24($29) + sd $19, 32($29) + sd $20, 40($29) + sd $21, 48($29) + sd $22, 56($29) + sd $23, 64($29) + sd $28, 72($29) + sd $30, 80($29) +#ifndef __mips_soft_float + sdc1 $f24, 88($29) + sdc1 $f25, 96($29) + sdc1 $f26, 104($29) + sdc1 $f27, 112($29) + sdc1 $f28, 120($29) + sdc1 $f29, 128($29) + sdc1 $f30, 136($29) + sdc1 $f31, 144($29) +#endif + + // prepare SB register = pc & 0xffffffff00000000 + bal 1f +1: + dsrl $28, $31, 32 + dsll $28, $28, 32 + + move $20, $4 // save R4 + move $1, $6 + jalr $5 // call setg_gcc (clobbers R4) + jalr $20 // call fn + + ld $16, 8($29) + ld $17, 16($29) + ld $18, 24($29) + ld $19, 32($29) + ld $20, 40($29) + ld $21, 48($29) + ld $22, 56($29) + ld $23, 64($29) + ld $28, 72($29) + ld $30, 80($29) +#ifndef __mips_soft_float + ldc1 $f24, 88($29) + ldc1 $f25, 96($29) + ldc1 $f26, 104($29) + ldc1 $f27, 112($29) + ldc1 $f28, 120($29) + ldc1 $f29, 128($29) + ldc1 $f30, 136($29) + ldc1 $f31, 144($29) +#endif + ld $31, 0($29) +#ifndef __mips_soft_float + daddiu $29, $29, 160 +#else + daddiu $29, $29, 96 +#endif + jr $31 + +.set at + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_mipsx.S b/src/runtime/cgo/gcc_mipsx.S new file mode 100644 index 0000000..54f4b82 --- /dev/null +++ b/src/runtime/cgo/gcc_mipsx.S @@ -0,0 +1,75 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +/* + * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard MIPS O32 ABI, where $16-$23, $30, and $f20-$f31 + * are callee-save, so they must be saved explicitly, along with $31 (LR). + */ +.globl crosscall1 +.set noat +crosscall1: +#ifndef __mips_soft_float + addiu $29, $29, -88 +#else + addiu $29, $29, -40 // For soft-float, no need to make room for FP registers +#endif + sw $31, 0($29) + sw $16, 4($29) + sw $17, 8($29) + sw $18, 12($29) + sw $19, 16($29) + sw $20, 20($29) + sw $21, 24($29) + sw $22, 28($29) + sw $23, 32($29) + sw $30, 36($29) + +#ifndef __mips_soft_float + sdc1 $f20, 40($29) + sdc1 $f22, 48($29) + sdc1 $f24, 56($29) + sdc1 $f26, 64($29) + sdc1 $f28, 72($29) + sdc1 $f30, 80($29) +#endif + move $20, $4 // save R4 + move $4, $6 + jalr $5 // call setg_gcc + jalr $20 // call fn + + lw $16, 4($29) + lw $17, 8($29) + lw $18, 12($29) + lw $19, 16($29) + lw $20, 20($29) + lw $21, 24($29) + lw $22, 28($29) + lw $23, 32($29) + lw $30, 36($29) +#ifndef __mips_soft_float + ldc1 $f20, 40($29) + ldc1 $f22, 48($29) + ldc1 $f24, 56($29) + ldc1 $f26, 64($29) + ldc1 $f28, 72($29) + ldc1 $f30, 80($29) +#endif + lw $31, 0($29) +#ifndef __mips_soft_float + addiu $29, $29, 88 +#else + addiu $29, $29, 40 +#endif + jr $31 + +.set at + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_mmap.c b/src/runtime/cgo/gcc_mmap.c new file mode 100644 index 0000000..698a7e3 --- /dev/null +++ b/src/runtime/cgo/gcc_mmap.c @@ -0,0 +1,39 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux,amd64 linux,arm64 linux,ppc64le + +#include <errno.h> +#include <stdint.h> +#include <stdlib.h> +#include <sys/mman.h> + +#include "libcgo.h" + +uintptr_t +x_cgo_mmap(void *addr, uintptr_t length, int32_t prot, int32_t flags, int32_t fd, uint32_t offset) { + void *p; + + _cgo_tsan_acquire(); + p = mmap(addr, length, prot, flags, fd, offset); + _cgo_tsan_release(); + if (p == MAP_FAILED) { + /* This is what the Go code expects on failure. */ + return (uintptr_t)errno; + } + return (uintptr_t)p; +} + +void +x_cgo_munmap(void *addr, uintptr_t length) { + int r; + + _cgo_tsan_acquire(); + r = munmap(addr, length); + _cgo_tsan_release(); + if (r < 0) { + /* The Go runtime is not prepared for munmap to fail. */ + abort(); + } +} diff --git a/src/runtime/cgo/gcc_netbsd_386.c b/src/runtime/cgo/gcc_netbsd_386.c new file mode 100644 index 0000000..5495f0f --- /dev/null +++ b/src/runtime/cgo/gcc_netbsd_386.c @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + stack_t ss; + + ts = *(ThreadStart*)v; + free(v); + + /* + * Set specific keys. + */ + setg_gcc((void*)ts.g); + + // On NetBSD, a new thread inherits the signal stack of the + // creating thread. That confuses minit, so we remove that + // signal stack here before calling the regular mstart. It's + // a bit baroque to remove a signal stack here only to add one + // in minit, but it's a simple change that keeps NetBSD + // working like other OS's. At this point all signals are + // blocked, so there is no race. + memset(&ss, 0, sizeof ss); + ss.ss_flags = SS_DISABLE; + sigaltstack(&ss, nil); + + crosscall_386(ts.fn); + return nil; +} diff --git a/src/runtime/cgo/gcc_netbsd_amd64.c b/src/runtime/cgo/gcc_netbsd_amd64.c new file mode 100644 index 0000000..9f4b031 --- /dev/null +++ b/src/runtime/cgo/gcc_netbsd_amd64.c @@ -0,0 +1,78 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + stack_t ss; + + ts = *(ThreadStart*)v; + free(v); + + // On NetBSD, a new thread inherits the signal stack of the + // creating thread. That confuses minit, so we remove that + // signal stack here before calling the regular mstart. It's + // a bit baroque to remove a signal stack here only to add one + // in minit, but it's a simple change that keeps NetBSD + // working like other OS's. At this point all signals are + // blocked, so there is no race. + memset(&ss, 0, sizeof ss); + ss.ss_flags = SS_DISABLE; + sigaltstack(&ss, nil); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_netbsd_arm.c b/src/runtime/cgo/gcc_netbsd_arm.c new file mode 100644 index 0000000..b0c80ea --- /dev/null +++ b/src/runtime/cgo/gcc_netbsd_arm.c @@ -0,0 +1,79 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall_arm1(void (*fn)(void), void (*setg_gcc)(void*), void *g); +static void* +threadentry(void *v) +{ + ThreadStart ts; + stack_t ss; + + ts = *(ThreadStart*)v; + free(v); + + // On NetBSD, a new thread inherits the signal stack of the + // creating thread. That confuses minit, so we remove that + // signal stack here before calling the regular mstart. It's + // a bit baroque to remove a signal stack here only to add one + // in minit, but it's a simple change that keeps NetBSD + // working like other OS's. At this point all signals are + // blocked, so there is no race. + memset(&ss, 0, sizeof ss); + ss.ss_flags = SS_DISABLE; + sigaltstack(&ss, nil); + + crosscall_arm1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_netbsd_arm64.c b/src/runtime/cgo/gcc_netbsd_arm64.c new file mode 100644 index 0000000..694116c --- /dev/null +++ b/src/runtime/cgo/gcc_netbsd_arm64.c @@ -0,0 +1,80 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + stack_t ss; + + ts = *(ThreadStart*)v; + free(v); + + // On NetBSD, a new thread inherits the signal stack of the + // creating thread. That confuses minit, so we remove that + // signal stack here before calling the regular mstart. It's + // a bit baroque to remove a signal stack here only to add one + // in minit, but it's a simple change that keeps NetBSD + // working like other OS's. At this point all signals are + // blocked, so there is no race. + memset(&ss, 0, sizeof ss); + ss.ss_flags = SS_DISABLE; + sigaltstack(&ss, nil); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_openbsd_386.c b/src/runtime/cgo/gcc_openbsd_386.c new file mode 100644 index 0000000..127a1b6 --- /dev/null +++ b/src/runtime/cgo/gcc_openbsd_386.c @@ -0,0 +1,70 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + /* + * Set specific keys. + */ + setg_gcc((void*)ts.g); + + crosscall_386(ts.fn); + return nil; +} diff --git a/src/runtime/cgo/gcc_openbsd_amd64.c b/src/runtime/cgo/gcc_openbsd_amd64.c new file mode 100644 index 0000000..09d2750 --- /dev/null +++ b/src/runtime/cgo/gcc_openbsd_amd64.c @@ -0,0 +1,65 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_openbsd_arm.c b/src/runtime/cgo/gcc_openbsd_arm.c new file mode 100644 index 0000000..9a5757f --- /dev/null +++ b/src/runtime/cgo/gcc_openbsd_arm.c @@ -0,0 +1,67 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall_arm1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_arm1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_openbsd_arm64.c b/src/runtime/cgo/gcc_openbsd_arm64.c new file mode 100644 index 0000000..abf9f66 --- /dev/null +++ b/src/runtime/cgo/gcc_openbsd_arm64.c @@ -0,0 +1,67 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_openbsd_mips64.c b/src/runtime/cgo/gcc_openbsd_mips64.c new file mode 100644 index 0000000..79f039a --- /dev/null +++ b/src/runtime/cgo/gcc_openbsd_mips64.c @@ -0,0 +1,67 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <sys/types.h> +#include <pthread.h> +#include <signal.h> +#include <string.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_ppc64x.c b/src/runtime/cgo/gcc_ppc64x.c new file mode 100644 index 0000000..9cb6e0c --- /dev/null +++ b/src/runtime/cgo/gcc_ppc64x.c @@ -0,0 +1,71 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void *threadentry(void*); + +void (*x_cgo_inittls)(void **tlsg, void **tlsbase); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsbase) +{ + pthread_attr_t attr; + size_t size; + + setg_gcc = setg; + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + g->stacklo = (uintptr)&attr - size + 4096; + pthread_attr_destroy(&attr); +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + pthread_attr_getstacksize(&attr, &size); + // Leave stacklo=0 and set stackhi=size; mstart will do the rest. + ts->g->stackhi = size; + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fatalf("pthread_create failed: %s", strerror(err)); + } +} + +extern void crosscall_ppc64(void (*fn)(void), void *g); + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + // Save g for this thread in C TLS + setg_gcc((void*)ts.g); + + crosscall_ppc64(ts.fn, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_riscv64.S b/src/runtime/cgo/gcc_riscv64.S new file mode 100644 index 0000000..f429dc6 --- /dev/null +++ b/src/runtime/cgo/gcc_riscv64.S @@ -0,0 +1,80 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * void crosscall1(void (*fn)(void), void (*setg_gcc)(void *g), void *g) + * + * Calling into the gc tool chain, where all registers are caller save. + * Called from standard RISCV ELF psABI, where x8-x9, x18-x27, f8-f9 and + * f18-f27 are callee-save, so they must be saved explicitly, along with + * x1 (LR). + */ +.globl crosscall1 +crosscall1: + sd x1, -200(sp) + addi sp, sp, -200 + sd x8, 8(sp) + sd x9, 16(sp) + sd x18, 24(sp) + sd x19, 32(sp) + sd x20, 40(sp) + sd x21, 48(sp) + sd x22, 56(sp) + sd x23, 64(sp) + sd x24, 72(sp) + sd x25, 80(sp) + sd x26, 88(sp) + sd x27, 96(sp) + fsd f8, 104(sp) + fsd f9, 112(sp) + fsd f18, 120(sp) + fsd f19, 128(sp) + fsd f20, 136(sp) + fsd f21, 144(sp) + fsd f22, 152(sp) + fsd f23, 160(sp) + fsd f24, 168(sp) + fsd f25, 176(sp) + fsd f26, 184(sp) + fsd f27, 192(sp) + + // a0 = *fn, a1 = *setg_gcc, a2 = *g + mv s1, a0 + mv s0, a1 + mv a0, a2 + jalr ra, s0 // call setg_gcc (clobbers x30 aka g) + jalr ra, s1 // call fn + + ld x1, 0(sp) + ld x8, 8(sp) + ld x9, 16(sp) + ld x18, 24(sp) + ld x19, 32(sp) + ld x20, 40(sp) + ld x21, 48(sp) + ld x22, 56(sp) + ld x23, 64(sp) + ld x24, 72(sp) + ld x25, 80(sp) + ld x26, 88(sp) + ld x27, 96(sp) + fld f8, 104(sp) + fld f9, 112(sp) + fld f18, 120(sp) + fld f19, 128(sp) + fld f20, 136(sp) + fld f21, 144(sp) + fld f22, 152(sp) + fld f23, 160(sp) + fld f24, 168(sp) + fld f25, 176(sp) + fld f26, 184(sp) + fld f27, 192(sp) + addi sp, sp, 200 + + jr ra + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_s390x.S b/src/runtime/cgo/gcc_s390x.S new file mode 100644 index 0000000..614de4b --- /dev/null +++ b/src/runtime/cgo/gcc_s390x.S @@ -0,0 +1,56 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * void crosscall_s390x(void (*fn)(void), void *g) + * + * Calling into the go tool chain, where all registers are caller save. + * Called from standard s390x C ABI, where r6-r13, r15, and f8-f15 are + * callee-save, so they must be saved explicitly. + */ +.globl crosscall_s390x +crosscall_s390x: + /* save r6-r15 in the register save area of the calling function */ + stmg %r6, %r15, 48(%r15) + + /* allocate 64 bytes of stack space to save f8-f15 */ + lay %r15, -64(%r15) + + /* save callee-saved floating point registers */ + std %f8, 0(%r15) + std %f9, 8(%r15) + std %f10, 16(%r15) + std %f11, 24(%r15) + std %f12, 32(%r15) + std %f13, 40(%r15) + std %f14, 48(%r15) + std %f15, 56(%r15) + + /* restore g pointer */ + lgr %r13, %r3 + + /* call fn */ + basr %r14, %r2 + + /* restore floating point registers */ + ld %f8, 0(%r15) + ld %f9, 8(%r15) + ld %f10, 16(%r15) + ld %f11, 24(%r15) + ld %f12, 32(%r15) + ld %f13, 40(%r15) + ld %f14, 48(%r15) + ld %f15, 56(%r15) + + /* de-allocate stack frame */ + la %r15, 64(%r15) + + /* restore general purpose registers */ + lmg %r6, %r15, 48(%r15) + + br %r14 /* restored by lmg */ + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif diff --git a/src/runtime/cgo/gcc_setenv.c b/src/runtime/cgo/gcc_setenv.c new file mode 100644 index 0000000..d4f7983 --- /dev/null +++ b/src/runtime/cgo/gcc_setenv.c @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris + +#include "libcgo.h" + +#include <stdlib.h> + +/* Stub for calling setenv */ +void +x_cgo_setenv(char **arg) +{ + _cgo_tsan_acquire(); + setenv(arg[0], arg[1], 1); + _cgo_tsan_release(); +} + +/* Stub for calling unsetenv */ +void +x_cgo_unsetenv(char **arg) +{ + _cgo_tsan_acquire(); + unsetenv(arg[0]); + _cgo_tsan_release(); +} diff --git a/src/runtime/cgo/gcc_sigaction.c b/src/runtime/cgo/gcc_sigaction.c new file mode 100644 index 0000000..fcf1e50 --- /dev/null +++ b/src/runtime/cgo/gcc_sigaction.c @@ -0,0 +1,82 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux,amd64 linux,arm64 linux,ppc64le + +#include <errno.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> +#include <signal.h> + +#include "libcgo.h" + +// go_sigaction_t is a C version of the sigactiont struct from +// defs_linux_amd64.go. This definition — and its conversion to and from struct +// sigaction — are specific to linux/amd64. +typedef struct { + uintptr_t handler; + uint64_t flags; + uintptr_t restorer; + uint64_t mask; +} go_sigaction_t; + +// SA_RESTORER is part of the kernel interface. +// This is Linux i386/amd64 specific. +#ifndef SA_RESTORER +#define SA_RESTORER 0x4000000 +#endif + +int32_t +x_cgo_sigaction(intptr_t signum, const go_sigaction_t *goact, go_sigaction_t *oldgoact) { + int32_t ret; + struct sigaction act; + struct sigaction oldact; + size_t i; + + _cgo_tsan_acquire(); + + memset(&act, 0, sizeof act); + memset(&oldact, 0, sizeof oldact); + + if (goact) { + if (goact->flags & SA_SIGINFO) { + act.sa_sigaction = (void(*)(int, siginfo_t*, void*))(goact->handler); + } else { + act.sa_handler = (void(*)(int))(goact->handler); + } + sigemptyset(&act.sa_mask); + for (i = 0; i < 8 * sizeof(goact->mask); i++) { + if (goact->mask & ((uint64_t)(1)<<i)) { + sigaddset(&act.sa_mask, (int)(i+1)); + } + } + act.sa_flags = (int)(goact->flags & ~(uint64_t)SA_RESTORER); + } + + ret = sigaction((int)signum, goact ? &act : NULL, oldgoact ? &oldact : NULL); + if (ret == -1) { + // runtime.rt_sigaction expects _cgo_sigaction to return errno on error. + _cgo_tsan_release(); + return errno; + } + + if (oldgoact) { + if (oldact.sa_flags & SA_SIGINFO) { + oldgoact->handler = (uintptr_t)(oldact.sa_sigaction); + } else { + oldgoact->handler = (uintptr_t)(oldact.sa_handler); + } + oldgoact->mask = 0; + for (i = 0; i < 8 * sizeof(oldgoact->mask); i++) { + if (sigismember(&oldact.sa_mask, (int)(i+1)) == 1) { + oldgoact->mask |= (uint64_t)(1)<<i; + } + } + oldgoact->flags = (uint64_t)oldact.sa_flags; + } + + _cgo_tsan_release(); + return ret; +} diff --git a/src/runtime/cgo/gcc_signal2_ios_arm64.c b/src/runtime/cgo/gcc_signal2_ios_arm64.c new file mode 100644 index 0000000..5b8a18f --- /dev/null +++ b/src/runtime/cgo/gcc_signal2_ios_arm64.c @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build lldb + +// Used by gcc_signal_darwin_arm64.c when doing the test build during cgo. +// We hope that for real binaries the definition provided by Go will take precedence +// and the linker will drop this .o file altogether, which is why this definition +// is all by itself in its own file. +void __attribute__((weak)) xx_cgo_panicmem(void) {} diff --git a/src/runtime/cgo/gcc_signal_ios_arm64.c b/src/runtime/cgo/gcc_signal_ios_arm64.c new file mode 100644 index 0000000..6519edd --- /dev/null +++ b/src/runtime/cgo/gcc_signal_ios_arm64.c @@ -0,0 +1,213 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Emulation of the Unix signal SIGSEGV. +// +// On iOS, Go tests and apps under development are run by lldb. +// The debugger uses a task-level exception handler to intercept signals. +// Despite having a 'handle' mechanism like gdb, lldb will not allow a +// SIGSEGV to pass to the running program. For Go, this means we cannot +// generate a panic, which cannot be recovered, and so tests fail. +// +// We work around this by registering a thread-level mach exception handler +// and intercepting EXC_BAD_ACCESS. The kernel offers thread handlers a +// chance to resolve exceptions before the task handler, so we can generate +// the panic and avoid lldb's SIGSEGV handler. +// +// The dist tool enables this by build flag when testing. + +// +build lldb + +#include <limits.h> +#include <pthread.h> +#include <stdio.h> +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +#include <mach/arm/thread_status.h> +#include <mach/exception_types.h> +#include <mach/mach.h> +#include <mach/mach_init.h> +#include <mach/mach_port.h> +#include <mach/thread_act.h> +#include <mach/thread_status.h> + +#include "libcgo.h" +#include "libcgo_unix.h" + +void xx_cgo_panicmem(void); +uintptr_t x_cgo_panicmem = (uintptr_t)xx_cgo_panicmem; + +static pthread_mutex_t mach_exception_handler_port_set_mu; +static mach_port_t mach_exception_handler_port_set = MACH_PORT_NULL; + +kern_return_t +catch_exception_raise( + mach_port_t exception_port, + mach_port_t thread, + mach_port_t task, + exception_type_t exception, + exception_data_t code_vector, + mach_msg_type_number_t code_count) +{ + kern_return_t ret; + arm_unified_thread_state_t thread_state; + mach_msg_type_number_t state_count = ARM_UNIFIED_THREAD_STATE_COUNT; + + // Returning KERN_SUCCESS intercepts the exception. + // + // Returning KERN_FAILURE lets the exception fall through to the + // next handler, which is the standard signal emulation code + // registered on the task port. + + if (exception != EXC_BAD_ACCESS) { + return KERN_FAILURE; + } + + ret = thread_get_state(thread, ARM_UNIFIED_THREAD_STATE, (thread_state_t)&thread_state, &state_count); + if (ret) { + fprintf(stderr, "runtime/cgo: thread_get_state failed: %d\n", ret); + abort(); + } + + // Bounce call to sigpanic through asm that makes it look like + // we call sigpanic directly from the faulting code. +#ifdef __arm64__ + thread_state.ts_64.__x[1] = thread_state.ts_64.__lr; + thread_state.ts_64.__x[2] = thread_state.ts_64.__pc; + thread_state.ts_64.__pc = x_cgo_panicmem; +#else + thread_state.ts_32.__r[1] = thread_state.ts_32.__lr; + thread_state.ts_32.__r[2] = thread_state.ts_32.__pc; + thread_state.ts_32.__pc = x_cgo_panicmem; +#endif + + if (0) { + // Useful debugging logic when panicmem is broken. + // + // Sends the first SIGSEGV and lets lldb catch the + // second one, avoiding a loop that locks up iOS + // devices requiring a hard reboot. + fprintf(stderr, "runtime/cgo: caught exc_bad_access\n"); + fprintf(stderr, "__lr = %llx\n", thread_state.ts_64.__lr); + fprintf(stderr, "__pc = %llx\n", thread_state.ts_64.__pc); + static int pass1 = 0; + if (pass1) { + return KERN_FAILURE; + } + pass1 = 1; + } + + ret = thread_set_state(thread, ARM_UNIFIED_THREAD_STATE, (thread_state_t)&thread_state, state_count); + if (ret) { + fprintf(stderr, "runtime/cgo: thread_set_state failed: %d\n", ret); + abort(); + } + + return KERN_SUCCESS; +} + +void +darwin_arm_init_thread_exception_port() +{ + // Called by each new OS thread to bind its EXC_BAD_ACCESS exception + // to mach_exception_handler_port_set. + int ret; + mach_port_t port = MACH_PORT_NULL; + + ret = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &port); + if (ret) { + fprintf(stderr, "runtime/cgo: mach_port_allocate failed: %d\n", ret); + abort(); + } + ret = mach_port_insert_right( + mach_task_self(), + port, + port, + MACH_MSG_TYPE_MAKE_SEND); + if (ret) { + fprintf(stderr, "runtime/cgo: mach_port_insert_right failed: %d\n", ret); + abort(); + } + + ret = thread_set_exception_ports( + mach_thread_self(), + EXC_MASK_BAD_ACCESS, + port, + EXCEPTION_DEFAULT, + THREAD_STATE_NONE); + if (ret) { + fprintf(stderr, "runtime/cgo: thread_set_exception_ports failed: %d\n", ret); + abort(); + } + + ret = pthread_mutex_lock(&mach_exception_handler_port_set_mu); + if (ret) { + fprintf(stderr, "runtime/cgo: pthread_mutex_lock failed: %d\n", ret); + abort(); + } + ret = mach_port_move_member( + mach_task_self(), + port, + mach_exception_handler_port_set); + if (ret) { + fprintf(stderr, "runtime/cgo: mach_port_move_member failed: %d\n", ret); + abort(); + } + ret = pthread_mutex_unlock(&mach_exception_handler_port_set_mu); + if (ret) { + fprintf(stderr, "runtime/cgo: pthread_mutex_unlock failed: %d\n", ret); + abort(); + } +} + +static void* +mach_exception_handler(void *port) +{ + // Calls catch_exception_raise. + extern boolean_t exc_server(); + mach_msg_server(exc_server, 2048, (mach_port_t)port, 0); + abort(); // never returns +} + +void +darwin_arm_init_mach_exception_handler() +{ + pthread_mutex_init(&mach_exception_handler_port_set_mu, NULL); + + // Called once per process to initialize a mach port server, listening + // for EXC_BAD_ACCESS thread exceptions. + int ret; + pthread_t thr = NULL; + pthread_attr_t attr; + sigset_t ign, oset; + + ret = mach_port_allocate( + mach_task_self(), + MACH_PORT_RIGHT_PORT_SET, + &mach_exception_handler_port_set); + if (ret) { + fprintf(stderr, "runtime/cgo: mach_port_allocate failed for port_set: %d\n", ret); + abort(); + } + + // Block all signals to the exception handler thread + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + // Start a thread to handle exceptions. + uintptr_t port_set = (uintptr_t)mach_exception_handler_port_set; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + ret = _cgo_try_pthread_create(&thr, &attr, mach_exception_handler, (void*)port_set); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (ret) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %d\n", ret); + abort(); + } + pthread_attr_destroy(&attr); +} diff --git a/src/runtime/cgo/gcc_signal_ios_nolldb.c b/src/runtime/cgo/gcc_signal_ios_nolldb.c new file mode 100644 index 0000000..cfa4025 --- /dev/null +++ b/src/runtime/cgo/gcc_signal_ios_nolldb.c @@ -0,0 +1,12 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !lldb +// +build ios +// +build arm64 + +#include <stdint.h> + +void darwin_arm_init_thread_exception_port() {} +void darwin_arm_init_mach_exception_handler() {} diff --git a/src/runtime/cgo/gcc_solaris_amd64.c b/src/runtime/cgo/gcc_solaris_amd64.c new file mode 100644 index 0000000..e89e844 --- /dev/null +++ b/src/runtime/cgo/gcc_solaris_amd64.c @@ -0,0 +1,77 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <pthread.h> +#include <string.h> +#include <signal.h> +#include <ucontext.h> +#include "libcgo.h" +#include "libcgo_unix.h" + +static void* threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + ucontext_t ctx; + + setg_gcc = setg; + if (getcontext(&ctx) != 0) + perror("runtime/cgo: getcontext failed"); + g->stacklo = (uintptr_t)ctx.uc_stack.ss_sp; + + // Solaris processes report a tiny stack when run with "ulimit -s unlimited". + // Correct that as best we can: assume it's at least 1 MB. + // See golang.org/issue/12210. + if(ctx.uc_stack.ss_size < 1024*1024) + g->stacklo -= 1024*1024 - ctx.uc_stack.ss_size; +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + pthread_attr_t attr; + sigset_t ign, oset; + pthread_t p; + void *base; + size_t size; + int err; + + sigfillset(&ign); + pthread_sigmask(SIG_SETMASK, &ign, &oset); + + pthread_attr_init(&attr); + + if (pthread_attr_getstack(&attr, &base, &size) != 0) + perror("runtime/cgo: pthread_attr_getstack failed"); + if (size == 0) { + ts->g->stackhi = 2 << 20; + if (pthread_attr_setstack(&attr, NULL, ts->g->stackhi) != 0) + perror("runtime/cgo: pthread_attr_setstack failed"); + } else { + ts->g->stackhi = size; + } + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + err = _cgo_try_pthread_create(&p, &attr, threadentry, ts); + + pthread_sigmask(SIG_SETMASK, &oset, nil); + + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } +} + +static void* +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); + return nil; +} diff --git a/src/runtime/cgo/gcc_traceback.c b/src/runtime/cgo/gcc_traceback.c new file mode 100644 index 0000000..6e9470c --- /dev/null +++ b/src/runtime/cgo/gcc_traceback.c @@ -0,0 +1,44 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo,darwin cgo,linux + +#include <stdint.h> +#include "libcgo.h" + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_feature(memory_sanitizer) +#include <sanitizer/msan_interface.h> +#endif + +// Call the user's traceback function and then call sigtramp. +// The runtime signal handler will jump to this code. +// We do it this way so that the user's traceback function will be called +// by a C function with proper unwind info. +void +x_cgo_callers(uintptr_t sig, void *info, void *context, void (*cgoTraceback)(struct cgoTracebackArg*), uintptr_t* cgoCallers, void (*sigtramp)(uintptr_t, void*, void*)) { + struct cgoTracebackArg arg; + + arg.Context = 0; + arg.SigContext = (uintptr_t)(context); + arg.Buf = cgoCallers; + arg.Max = 32; // must match len(runtime.cgoCallers) + +#if __has_feature(memory_sanitizer) + // This function is called directly from the signal handler. + // The arguments are passed in registers, so whether msan + // considers cgoCallers to be initialized depends on whether + // it considers the appropriate register to be initialized. + // That can cause false reports in rare cases. + // Explicitly unpoison the memory to avoid that. + // See issue #47543 for more details. + __msan_unpoison(&arg, sizeof arg); +#endif + + (*cgoTraceback)(&arg); + sigtramp(sig, info, context); +} diff --git a/src/runtime/cgo/gcc_util.c b/src/runtime/cgo/gcc_util.c new file mode 100644 index 0000000..3fcb48c --- /dev/null +++ b/src/runtime/cgo/gcc_util.c @@ -0,0 +1,69 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "libcgo.h" + +/* Stub for creating a new thread */ +void +x_cgo_thread_start(ThreadStart *arg) +{ + ThreadStart *ts; + + /* Make our own copy that can persist after we return. */ + _cgo_tsan_acquire(); + ts = malloc(sizeof *ts); + _cgo_tsan_release(); + if(ts == nil) { + fprintf(stderr, "runtime/cgo: out of memory in thread_start\n"); + abort(); + } + *ts = *arg; + + _cgo_sys_thread_start(ts); /* OS-dependent half */ +} + +#ifndef CGO_TSAN +void(* const _cgo_yield)() = NULL; +#else + +#include <string.h> + +char x_cgo_yield_strncpy_src = 0; +char x_cgo_yield_strncpy_dst = 0; +size_t x_cgo_yield_strncpy_n = 0; + +/* +Stub for allowing libc interceptors to execute. + +_cgo_yield is set to NULL if we do not expect libc interceptors to exist. +*/ +static void +x_cgo_yield() +{ + /* + The libc function(s) we call here must form a no-op and include at least one + call that triggers TSAN to process pending asynchronous signals. + + sleep(0) would be fine, but it's not portable C (so it would need more header + guards). + free(NULL) has a fast-path special case in TSAN, so it doesn't + trigger signal delivery. + free(malloc(0)) would work (triggering the interceptors in malloc), but + it also runs a bunch of user-supplied malloc hooks. + + So we choose strncpy(_, _, 0): it requires an extra header, + but it's standard and should be very efficient. + + GCC 7 has an unfortunate habit of optimizing out strncpy calls (see + https://golang.org/issue/21196), so the arguments here need to be global + variables with external linkage in order to ensure that the call traps all the + way down into libc. + */ + strncpy(&x_cgo_yield_strncpy_dst, &x_cgo_yield_strncpy_src, + x_cgo_yield_strncpy_n); +} + +void(* const _cgo_yield)() = &x_cgo_yield; + +#endif /* GO_TSAN */ diff --git a/src/runtime/cgo/gcc_windows_386.c b/src/runtime/cgo/gcc_windows_386.c new file mode 100644 index 0000000..60cb011 --- /dev/null +++ b/src/runtime/cgo/gcc_windows_386.c @@ -0,0 +1,55 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include "libcgo.h" +#include "libcgo_windows.h" + +static void threadentry(void*); + +void +x_cgo_init(G *g) +{ +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + uintptr_t thandle; + + thandle = _beginthread(threadentry, 0, ts); + if(thandle == -1) { + fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); + abort(); + } +} + +static void +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + // minit queries stack bounds from the OS. + + /* + * Set specific keys in thread local storage. + */ + asm volatile ( + "movl %0, %%fs:0x14\n" // MOVL tls0, 0x14(FS) + "movl %%fs:0x14, %%eax\n" // MOVL 0x14(FS), tmp + "movl %1, 0(%%eax)\n" // MOVL g, 0(FS) + :: "r"(ts.tls), "r"(ts.g) : "%eax" + ); + + crosscall_386(ts.fn); +} diff --git a/src/runtime/cgo/gcc_windows_amd64.c b/src/runtime/cgo/gcc_windows_amd64.c new file mode 100644 index 0000000..9df9b9b --- /dev/null +++ b/src/runtime/cgo/gcc_windows_amd64.c @@ -0,0 +1,55 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include "libcgo.h" +#include "libcgo_windows.h" + +static void threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*), void **tlsg, void **tlsbase) +{ + setg_gcc = setg; +} + + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + uintptr_t thandle; + + thandle = _beginthread(threadentry, 0, ts); + if(thandle == -1) { + fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); + abort(); + } +} + +static void +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + // minit queries stack bounds from the OS. + + /* + * Set specific keys in thread local storage. + */ + asm volatile ( + "movq %0, %%gs:0x28\n" // MOVL tls0, 0x28(GS) + :: "r"(ts.tls) + ); + + crosscall_amd64(ts.fn, setg_gcc, (void*)ts.g); +} diff --git a/src/runtime/cgo/gcc_windows_arm64.c b/src/runtime/cgo/gcc_windows_arm64.c new file mode 100644 index 0000000..61ef094 --- /dev/null +++ b/src/runtime/cgo/gcc_windows_arm64.c @@ -0,0 +1,46 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include "libcgo.h" +#include "libcgo_windows.h" + +static void threadentry(void*); +static void (*setg_gcc)(void*); + +void +x_cgo_init(G *g, void (*setg)(void*)) +{ + setg_gcc = setg; +} + +void +_cgo_sys_thread_start(ThreadStart *ts) +{ + uintptr_t thandle; + + thandle = _beginthread(threadentry, 0, ts); + if(thandle == -1) { + fprintf(stderr, "runtime: failed to create new OS thread (%d)\n", errno); + abort(); + } +} + +extern void crosscall1(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +static void +threadentry(void *v) +{ + ThreadStart ts; + + ts = *(ThreadStart*)v; + free(v); + + crosscall1(ts.fn, setg_gcc, (void *)ts.g); +} diff --git a/src/runtime/cgo/handle.go b/src/runtime/cgo/handle.go new file mode 100644 index 0000000..d711900 --- /dev/null +++ b/src/runtime/cgo/handle.go @@ -0,0 +1,144 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +import ( + "sync" + "sync/atomic" +) + +// Handle provides a way to pass values that contain Go pointers +// (pointers to memory allocated by Go) between Go and C without +// breaking the cgo pointer passing rules. A Handle is an integer +// value that can represent any Go value. A Handle can be passed +// through C and back to Go, and Go code can use the Handle to +// retrieve the original Go value. +// +// The underlying type of Handle is guaranteed to fit in an integer type +// that is large enough to hold the bit pattern of any pointer. The zero +// value of a Handle is not valid, and thus is safe to use as a sentinel +// in C APIs. +// +// For instance, on the Go side: +// +// package main +// +// /* +// #include <stdint.h> // for uintptr_t +// +// extern void MyGoPrint(uintptr_t handle); +// void myprint(uintptr_t handle); +// */ +// import "C" +// import "runtime/cgo" +// +// //export MyGoPrint +// func MyGoPrint(handle C.uintptr_t) { +// h := cgo.Handle(handle) +// val := h.Value().(string) +// println(val) +// h.Delete() +// } +// +// func main() { +// val := "hello Go" +// C.myprint(C.uintptr_t(cgo.NewHandle(val))) +// // Output: hello Go +// } +// +// and on the C side: +// +// #include <stdint.h> // for uintptr_t +// +// // A Go function +// extern void MyGoPrint(uintptr_t handle); +// +// // A C function +// void myprint(uintptr_t handle) { +// MyGoPrint(handle); +// } +// +// Some C functions accept a void* argument that points to an arbitrary +// data value supplied by the caller. It is not safe to coerce a cgo.Handle +// (an integer) to a Go unsafe.Pointer, but instead we can pass the address +// of the cgo.Handle to the void* parameter, as in this variant of the +// previous example: +// +// package main +// +// /* +// extern void MyGoPrint(void *context); +// static inline void myprint(void *context) { +// MyGoPrint(context); +// } +// */ +// import "C" +// import ( +// "runtime/cgo" +// "unsafe" +// ) +// +// //export MyGoPrint +// func MyGoPrint(context unsafe.Pointer) { +// h := *(*cgo.Handle)(context) +// val := h.Value().(string) +// println(val) +// h.Delete() +// } +// +// func main() { +// val := "hello Go" +// h := cgo.NewHandle(val) +// C.myprint(unsafe.Pointer(&h)) +// // Output: hello Go +// } +type Handle uintptr + +// NewHandle returns a handle for a given value. +// +// The handle is valid until the program calls Delete on it. The handle +// uses resources, and this package assumes that C code may hold on to +// the handle, so a program must explicitly call Delete when the handle +// is no longer needed. +// +// The intended use is to pass the returned handle to C code, which +// passes it back to Go, which calls Value. +func NewHandle(v any) Handle { + h := atomic.AddUintptr(&handleIdx, 1) + if h == 0 { + panic("runtime/cgo: ran out of handle space") + } + + handles.Store(h, v) + return Handle(h) +} + +// Value returns the associated Go value for a valid handle. +// +// The method panics if the handle is invalid. +func (h Handle) Value() any { + v, ok := handles.Load(uintptr(h)) + if !ok { + panic("runtime/cgo: misuse of an invalid Handle") + } + return v +} + +// Delete invalidates a handle. This method should only be called once +// the program no longer needs to pass the handle to C and the C code +// no longer has a copy of the handle value. +// +// The method panics if the handle is invalid. +func (h Handle) Delete() { + _, ok := handles.LoadAndDelete(uintptr(h)) + if !ok { + panic("runtime/cgo: misuse of an invalid Handle") + } +} + +var ( + handles = sync.Map{} // map[Handle]interface{} + handleIdx uintptr // atomic +) diff --git a/src/runtime/cgo/handle_test.go b/src/runtime/cgo/handle_test.go new file mode 100644 index 0000000..b341c8e --- /dev/null +++ b/src/runtime/cgo/handle_test.go @@ -0,0 +1,103 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +import ( + "reflect" + "testing" +) + +func TestHandle(t *testing.T) { + v := 42 + + tests := []struct { + v1 any + v2 any + }{ + {v1: v, v2: v}, + {v1: &v, v2: &v}, + {v1: nil, v2: nil}, + } + + for _, tt := range tests { + h1 := NewHandle(tt.v1) + h2 := NewHandle(tt.v2) + + if uintptr(h1) == 0 || uintptr(h2) == 0 { + t.Fatalf("NewHandle returns zero") + } + + if uintptr(h1) == uintptr(h2) { + t.Fatalf("Duplicated Go values should have different handles, but got equal") + } + + h1v := h1.Value() + h2v := h2.Value() + if !reflect.DeepEqual(h1v, h2v) || !reflect.DeepEqual(h1v, tt.v1) { + t.Fatalf("Value of a Handle got wrong, got %+v %+v, want %+v", h1v, h2v, tt.v1) + } + + h1.Delete() + h2.Delete() + } + + siz := 0 + handles.Range(func(k, v any) bool { + siz++ + return true + }) + if siz != 0 { + t.Fatalf("handles are not cleared, got %d, want %d", siz, 0) + } +} + +func TestInvalidHandle(t *testing.T) { + t.Run("zero", func(t *testing.T) { + h := Handle(0) + + defer func() { + if r := recover(); r != nil { + return + } + t.Fatalf("Delete of zero handle did not trigger a panic") + }() + + h.Delete() + }) + + t.Run("invalid", func(t *testing.T) { + h := NewHandle(42) + + defer func() { + if r := recover(); r != nil { + h.Delete() + return + } + t.Fatalf("Invalid handle did not trigger a panic") + }() + + Handle(h + 1).Delete() + }) +} + +func BenchmarkHandle(b *testing.B) { + b.Run("non-concurrent", func(b *testing.B) { + for i := 0; i < b.N; i++ { + h := NewHandle(i) + _ = h.Value() + h.Delete() + } + }) + b.Run("concurrent", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + var v int + for pb.Next() { + h := NewHandle(v) + _ = h.Value() + h.Delete() + } + }) + }) +} diff --git a/src/runtime/cgo/iscgo.go b/src/runtime/cgo/iscgo.go new file mode 100644 index 0000000..e12d0f4 --- /dev/null +++ b/src/runtime/cgo/iscgo.go @@ -0,0 +1,17 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The runtime package contains an uninitialized definition +// for runtime·iscgo. Override it to tell the runtime we're here. +// There are various function pointers that should be set too, +// but those depend on dynamic linker magic to get initialized +// correctly, and sometimes they break. This variable is a +// backup: it depends only on old C style static linking rules. + +package cgo + +import _ "unsafe" // for go:linkname + +//go:linkname _iscgo runtime.iscgo +var _iscgo bool = true diff --git a/src/runtime/cgo/libcgo.h b/src/runtime/cgo/libcgo.h new file mode 100644 index 0000000..af4960e --- /dev/null +++ b/src/runtime/cgo/libcgo.h @@ -0,0 +1,151 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> + +#undef nil +#define nil ((void*)0) +#define nelem(x) (sizeof(x)/sizeof((x)[0])) + +typedef uint32_t uint32; +typedef uint64_t uint64; +typedef uintptr_t uintptr; + +/* + * The beginning of the per-goroutine structure, + * as defined in ../pkg/runtime/runtime.h. + * Just enough to edit these two fields. + */ +typedef struct G G; +struct G +{ + uintptr stacklo; + uintptr stackhi; +}; + +/* + * Arguments to the _cgo_thread_start call. + * Also known to ../pkg/runtime/runtime.h. + */ +typedef struct ThreadStart ThreadStart; +struct ThreadStart +{ + G *g; + uintptr *tls; + void (*fn)(void); +}; + +/* + * Called by 5c/6c/8c world. + * Makes a local copy of the ThreadStart and + * calls _cgo_sys_thread_start(ts). + */ +extern void (*_cgo_thread_start)(ThreadStart *ts); + +/* + * Creates a new operating system thread without updating any Go state + * (OS dependent). + */ +extern void (*_cgo_sys_thread_create)(void* (*func)(void*), void* arg); + +/* + * Creates the new operating system thread (OS, arch dependent). + */ +void _cgo_sys_thread_start(ThreadStart *ts); + +/* + * Waits for the Go runtime to be initialized (OS dependent). + * If runtime.SetCgoTraceback is used to set a context function, + * calls the context function and returns the context value. + */ +uintptr_t _cgo_wait_runtime_init_done(void); + +/* + * Call fn in the 6c world. + */ +void crosscall_amd64(void (*fn)(void), void (*setg_gcc)(void*), void *g); + +/* + * Call fn in the 8c world. + */ +void crosscall_386(void (*fn)(void)); + +/* + * Prints error then calls abort. For linux and android. + */ +void fatalf(const char* format, ...); + +/* + * Registers the current mach thread port for EXC_BAD_ACCESS processing. + */ +void darwin_arm_init_thread_exception_port(void); + +/* + * Starts a mach message server processing EXC_BAD_ACCESS. + */ +void darwin_arm_init_mach_exception_handler(void); + +/* + * The cgo context function. See runtime.SetCgoTraceback. + */ +struct context_arg { + uintptr_t Context; +}; +extern void (*(_cgo_get_context_function(void)))(struct context_arg*); + +/* + * The argument for the cgo traceback callback. See runtime.SetCgoTraceback. + */ +struct cgoTracebackArg { + uintptr_t Context; + uintptr_t SigContext; + uintptr_t* Buf; + uintptr_t Max; +}; + +/* + * TSAN support. This is only useful when building with + * CGO_CFLAGS="-fsanitize=thread" CGO_LDFLAGS="-fsanitize=thread" go install + */ +#undef CGO_TSAN +#if defined(__has_feature) +# if __has_feature(thread_sanitizer) +# define CGO_TSAN +# endif +#elif defined(__SANITIZE_THREAD__) +# define CGO_TSAN +#endif + +#ifdef CGO_TSAN + +// These must match the definitions in yesTsanProlog in cmd/cgo/out.go. +// In general we should call _cgo_tsan_acquire when we enter C code, +// and call _cgo_tsan_release when we return to Go code. +// This is only necessary when calling code that might be instrumented +// by TSAN, which mostly means system library calls that TSAN intercepts. +// See the comment in cmd/cgo/out.go for more details. + +long long _cgo_sync __attribute__ ((common)); + +extern void __tsan_acquire(void*); +extern void __tsan_release(void*); + +__attribute__ ((unused)) +static void _cgo_tsan_acquire() { + __tsan_acquire(&_cgo_sync); +} + +__attribute__ ((unused)) +static void _cgo_tsan_release() { + __tsan_release(&_cgo_sync); +} + +#else // !defined(CGO_TSAN) + +#define _cgo_tsan_acquire() +#define _cgo_tsan_release() + +#endif // !defined(CGO_TSAN) diff --git a/src/runtime/cgo/libcgo_unix.h b/src/runtime/cgo/libcgo_unix.h new file mode 100644 index 0000000..a56a366 --- /dev/null +++ b/src/runtime/cgo/libcgo_unix.h @@ -0,0 +1,15 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + * Call pthread_create, retrying on EAGAIN. + */ +extern int _cgo_try_pthread_create(pthread_t*, const pthread_attr_t*, void* (*)(void*), void*); + +/* + * Same as _cgo_try_pthread_create, but passing on the pthread_create function. + * Only defined on OpenBSD. + */ +extern int _cgo_openbsd_try_pthread_create(int (*)(pthread_t*, const pthread_attr_t*, void *(*pfn)(void*), void*), + pthread_t*, const pthread_attr_t*, void* (*)(void*), void* arg); diff --git a/src/runtime/cgo/libcgo_windows.h b/src/runtime/cgo/libcgo_windows.h new file mode 100644 index 0000000..0013f06 --- /dev/null +++ b/src/runtime/cgo/libcgo_windows.h @@ -0,0 +1,12 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Ensure there's one symbol marked __declspec(dllexport). +// If there are no exported symbols, the unfortunate behavior of +// the binutils linker is to also strip the relocations table, +// resulting in non-PIE binary. The other option is the +// --export-all-symbols flag, but we don't need to export all symbols +// and this may overflow the export table (#40795). +// See https://sourceware.org/bugzilla/show_bug.cgi?id=19011 +__declspec(dllexport) int _cgo_dummy_export; diff --git a/src/runtime/cgo/linux.go b/src/runtime/cgo/linux.go new file mode 100644 index 0000000..1d6fe03 --- /dev/null +++ b/src/runtime/cgo/linux.go @@ -0,0 +1,74 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Linux system call wrappers that provide POSIX semantics through the +// corresponding cgo->libc (nptl) wrappers for various system calls. + +//go:build linux + +package cgo + +import "unsafe" + +// Each of the following entries is needed to ensure that the +// syscall.syscall_linux code can conditionally call these +// function pointers: +// +// 1. find the C-defined function start +// 2. force the local byte alias to be mapped to that location +// 3. map the Go pointer to the function to the syscall package + +//go:cgo_import_static _cgo_libc_setegid +//go:linkname _cgo_libc_setegid _cgo_libc_setegid +//go:linkname cgo_libc_setegid syscall.cgo_libc_setegid +var _cgo_libc_setegid byte +var cgo_libc_setegid = unsafe.Pointer(&_cgo_libc_setegid) + +//go:cgo_import_static _cgo_libc_seteuid +//go:linkname _cgo_libc_seteuid _cgo_libc_seteuid +//go:linkname cgo_libc_seteuid syscall.cgo_libc_seteuid +var _cgo_libc_seteuid byte +var cgo_libc_seteuid = unsafe.Pointer(&_cgo_libc_seteuid) + +//go:cgo_import_static _cgo_libc_setregid +//go:linkname _cgo_libc_setregid _cgo_libc_setregid +//go:linkname cgo_libc_setregid syscall.cgo_libc_setregid +var _cgo_libc_setregid byte +var cgo_libc_setregid = unsafe.Pointer(&_cgo_libc_setregid) + +//go:cgo_import_static _cgo_libc_setresgid +//go:linkname _cgo_libc_setresgid _cgo_libc_setresgid +//go:linkname cgo_libc_setresgid syscall.cgo_libc_setresgid +var _cgo_libc_setresgid byte +var cgo_libc_setresgid = unsafe.Pointer(&_cgo_libc_setresgid) + +//go:cgo_import_static _cgo_libc_setresuid +//go:linkname _cgo_libc_setresuid _cgo_libc_setresuid +//go:linkname cgo_libc_setresuid syscall.cgo_libc_setresuid +var _cgo_libc_setresuid byte +var cgo_libc_setresuid = unsafe.Pointer(&_cgo_libc_setresuid) + +//go:cgo_import_static _cgo_libc_setreuid +//go:linkname _cgo_libc_setreuid _cgo_libc_setreuid +//go:linkname cgo_libc_setreuid syscall.cgo_libc_setreuid +var _cgo_libc_setreuid byte +var cgo_libc_setreuid = unsafe.Pointer(&_cgo_libc_setreuid) + +//go:cgo_import_static _cgo_libc_setgroups +//go:linkname _cgo_libc_setgroups _cgo_libc_setgroups +//go:linkname cgo_libc_setgroups syscall.cgo_libc_setgroups +var _cgo_libc_setgroups byte +var cgo_libc_setgroups = unsafe.Pointer(&_cgo_libc_setgroups) + +//go:cgo_import_static _cgo_libc_setgid +//go:linkname _cgo_libc_setgid _cgo_libc_setgid +//go:linkname cgo_libc_setgid syscall.cgo_libc_setgid +var _cgo_libc_setgid byte +var cgo_libc_setgid = unsafe.Pointer(&_cgo_libc_setgid) + +//go:cgo_import_static _cgo_libc_setuid +//go:linkname _cgo_libc_setuid _cgo_libc_setuid +//go:linkname cgo_libc_setuid syscall.cgo_libc_setuid +var _cgo_libc_setuid byte +var cgo_libc_setuid = unsafe.Pointer(&_cgo_libc_setuid) diff --git a/src/runtime/cgo/linux_syscall.c b/src/runtime/cgo/linux_syscall.c new file mode 100644 index 0000000..59761c8 --- /dev/null +++ b/src/runtime/cgo/linux_syscall.c @@ -0,0 +1,85 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +#ifndef _GNU_SOURCE // setres[ug]id() API. +#define _GNU_SOURCE +#endif + +#include <grp.h> +#include <sys/types.h> +#include <unistd.h> +#include <errno.h> +#include "libcgo.h" + +/* + * Assumed POSIX compliant libc system call wrappers. For linux, the + * glibc/nptl/setxid mechanism ensures that POSIX semantics are + * honored for all pthreads (by default), and this in turn with cgo + * ensures that all Go threads launched with cgo are kept in sync for + * these function calls. + */ + +// argset_t matches runtime/cgocall.go:argset. +typedef struct { + uintptr_t* args; + uintptr_t retval; +} argset_t; + +// libc backed posix-compliant syscalls. + +#define SET_RETVAL(fn) \ + uintptr_t ret = (uintptr_t) fn ; \ + if (ret == (uintptr_t) -1) { \ + x->retval = (uintptr_t) errno; \ + } else \ + x->retval = ret + +void +_cgo_libc_setegid(argset_t* x) { + SET_RETVAL(setegid((gid_t) x->args[0])); +} + +void +_cgo_libc_seteuid(argset_t* x) { + SET_RETVAL(seteuid((uid_t) x->args[0])); +} + +void +_cgo_libc_setgid(argset_t* x) { + SET_RETVAL(setgid((gid_t) x->args[0])); +} + +void +_cgo_libc_setgroups(argset_t* x) { + SET_RETVAL(setgroups((size_t) x->args[0], (const gid_t *) x->args[1])); +} + +void +_cgo_libc_setregid(argset_t* x) { + SET_RETVAL(setregid((gid_t) x->args[0], (gid_t) x->args[1])); +} + +void +_cgo_libc_setresgid(argset_t* x) { + SET_RETVAL(setresgid((gid_t) x->args[0], (gid_t) x->args[1], + (gid_t) x->args[2])); +} + +void +_cgo_libc_setresuid(argset_t* x) { + SET_RETVAL(setresuid((uid_t) x->args[0], (uid_t) x->args[1], + (uid_t) x->args[2])); +} + +void +_cgo_libc_setreuid(argset_t* x) { + SET_RETVAL(setreuid((uid_t) x->args[0], (uid_t) x->args[1])); +} + +void +_cgo_libc_setuid(argset_t* x) { + SET_RETVAL(setuid((uid_t) x->args[0])); +} diff --git a/src/runtime/cgo/mmap.go b/src/runtime/cgo/mmap.go new file mode 100644 index 0000000..eae0a9e --- /dev/null +++ b/src/runtime/cgo/mmap.go @@ -0,0 +1,31 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (linux && amd64) || (linux && arm64) + +package cgo + +// Import "unsafe" because we use go:linkname. +import _ "unsafe" + +// When using cgo, call the C library for mmap, so that we call into +// any sanitizer interceptors. This supports using the memory +// sanitizer with Go programs. The memory sanitizer only applies to +// C/C++ code; this permits that code to see the Go code as normal +// program addresses that have been initialized. + +// To support interceptors that look for both mmap and munmap, +// also call the C library for munmap. + +//go:cgo_import_static x_cgo_mmap +//go:linkname x_cgo_mmap x_cgo_mmap +//go:linkname _cgo_mmap _cgo_mmap +var x_cgo_mmap byte +var _cgo_mmap = &x_cgo_mmap + +//go:cgo_import_static x_cgo_munmap +//go:linkname x_cgo_munmap x_cgo_munmap +//go:linkname _cgo_munmap _cgo_munmap +var x_cgo_munmap byte +var _cgo_munmap = &x_cgo_munmap diff --git a/src/runtime/cgo/netbsd.go b/src/runtime/cgo/netbsd.go new file mode 100644 index 0000000..8a8018b --- /dev/null +++ b/src/runtime/cgo/netbsd.go @@ -0,0 +1,21 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build netbsd + +package cgo + +import _ "unsafe" // for go:linkname + +// Supply environ and __progname, because we don't +// link against the standard NetBSD crt0.o and the +// libc dynamic library needs them. + +//go:linkname _environ environ +//go:linkname _progname __progname +//go:linkname ___ps_strings __ps_strings + +var _environ uintptr +var _progname uintptr +var ___ps_strings uintptr diff --git a/src/runtime/cgo/openbsd.go b/src/runtime/cgo/openbsd.go new file mode 100644 index 0000000..872d02e --- /dev/null +++ b/src/runtime/cgo/openbsd.go @@ -0,0 +1,20 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build openbsd + +package cgo + +import _ "unsafe" // for go:linkname + +// Supply __guard_local because we don't link against the standard +// OpenBSD crt0.o and the libc dynamic library needs it. + +//go:linkname _guard_local __guard_local + +var _guard_local uintptr + +// This is normally marked as hidden and placed in the +// .openbsd.randomdata section. +//go:cgo_export_dynamic __guard_local __guard_local diff --git a/src/runtime/cgo/setenv.go b/src/runtime/cgo/setenv.go new file mode 100644 index 0000000..0f4780a --- /dev/null +++ b/src/runtime/cgo/setenv.go @@ -0,0 +1,21 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package cgo + +import _ "unsafe" // for go:linkname + +//go:cgo_import_static x_cgo_setenv +//go:linkname x_cgo_setenv x_cgo_setenv +//go:linkname _cgo_setenv runtime._cgo_setenv +var x_cgo_setenv byte +var _cgo_setenv = &x_cgo_setenv + +//go:cgo_import_static x_cgo_unsetenv +//go:linkname x_cgo_unsetenv x_cgo_unsetenv +//go:linkname _cgo_unsetenv runtime._cgo_unsetenv +var x_cgo_unsetenv byte +var _cgo_unsetenv = &x_cgo_unsetenv diff --git a/src/runtime/cgo/sigaction.go b/src/runtime/cgo/sigaction.go new file mode 100644 index 0000000..dc714f7 --- /dev/null +++ b/src/runtime/cgo/sigaction.go @@ -0,0 +1,22 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (linux && amd64) || (freebsd && amd64) || (linux && arm64) || (linux && ppc64le) + +package cgo + +// Import "unsafe" because we use go:linkname. +import _ "unsafe" + +// When using cgo, call the C library for sigaction, so that we call into +// any sanitizer interceptors. This supports using the sanitizers +// with Go programs. The thread and memory sanitizers only apply to +// C/C++ code; this permits that code to see the Go runtime's existing signal +// handlers when registering new signal handlers for the process. + +//go:cgo_import_static x_cgo_sigaction +//go:linkname x_cgo_sigaction x_cgo_sigaction +//go:linkname _cgo_sigaction _cgo_sigaction +var x_cgo_sigaction byte +var _cgo_sigaction = &x_cgo_sigaction diff --git a/src/runtime/cgo/signal_ios_arm64.go b/src/runtime/cgo/signal_ios_arm64.go new file mode 100644 index 0000000..3425c44 --- /dev/null +++ b/src/runtime/cgo/signal_ios_arm64.go @@ -0,0 +1,10 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgo + +import _ "unsafe" + +//go:cgo_export_static xx_cgo_panicmem xx_cgo_panicmem +func xx_cgo_panicmem() diff --git a/src/runtime/cgo/signal_ios_arm64.s b/src/runtime/cgo/signal_ios_arm64.s new file mode 100644 index 0000000..1ae00d1 --- /dev/null +++ b/src/runtime/cgo/signal_ios_arm64.s @@ -0,0 +1,56 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// xx_cgo_panicmem is the entrypoint for SIGSEGV as intercepted via a +// mach thread port as EXC_BAD_ACCESS. As the segfault may have happened +// in C code, we first need to load_g then call xx_cgo_panicmem. +// +// R1 - LR at moment of fault +// R2 - PC at moment of fault +TEXT xx_cgo_panicmem(SB),NOSPLIT|NOFRAME,$0 + // If in external C code, we need to load the g register. + BL runtime·load_g(SB) + CMP $0, g + BNE ongothread + + // On a foreign thread. + // TODO(crawshaw): call badsignal + MOVD.W $0, -16(RSP) + MOVW $139, R1 + MOVW R1, 8(RSP) + B runtime·exit(SB) + +ongothread: + // Trigger a SIGSEGV panic. + // + // The goal is to arrange the stack so it looks like the runtime + // function sigpanic was called from the PC that faulted. It has + // to be sigpanic, as the stack unwinding code in traceback.go + // looks explicitly for it. + // + // To do this we call into runtime·setsigsegv, which sets the + // appropriate state inside the g object. We give it the faulting + // PC on the stack, then put it in the LR before calling sigpanic. + + // Build a 32-byte stack frame for us for this call. + // Saved LR (none available) is at the bottom, + // then the PC argument for setsigsegv, + // then a copy of the LR for us to restore. + MOVD.W $0, -32(RSP) + MOVD R1, 8(RSP) + MOVD R2, 16(RSP) + BL runtime·setsigsegv(SB) + MOVD 8(RSP), R1 + MOVD 16(RSP), R2 + + // Build a 16-byte stack frame for the simulated + // call to sigpanic, by taking 16 bytes away from the + // 32-byte stack frame above. + // The saved LR in this frame is the LR at time of fault, + // and the LR on entry to sigpanic is the PC at time of fault. + MOVD.W R1, 16(RSP) + MOVD R2, R30 + B runtime·sigpanic(SB) diff --git a/src/runtime/cgo_mmap.go b/src/runtime/cgo_mmap.go new file mode 100644 index 0000000..0cb25bd --- /dev/null +++ b/src/runtime/cgo_mmap.go @@ -0,0 +1,67 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Support for memory sanitizer. See runtime/cgo/mmap.go. + +//go:build (linux && amd64) || (linux && arm64) + +package runtime + +import "unsafe" + +// _cgo_mmap is filled in by runtime/cgo when it is linked into the +// program, so it is only non-nil when using cgo. +//go:linkname _cgo_mmap _cgo_mmap +var _cgo_mmap unsafe.Pointer + +// _cgo_munmap is filled in by runtime/cgo when it is linked into the +// program, so it is only non-nil when using cgo. +//go:linkname _cgo_munmap _cgo_munmap +var _cgo_munmap unsafe.Pointer + +// mmap is used to route the mmap system call through C code when using cgo, to +// support sanitizer interceptors. Don't allow stack splits, since this function +// (used by sysAlloc) is called in a lot of low-level parts of the runtime and +// callers often assume it won't acquire any locks. +//go:nosplit +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { + if _cgo_mmap != nil { + // Make ret a uintptr so that writing to it in the + // function literal does not trigger a write barrier. + // A write barrier here could break because of the way + // that mmap uses the same value both as a pointer and + // an errno value. + var ret uintptr + systemstack(func() { + ret = callCgoMmap(addr, n, prot, flags, fd, off) + }) + if ret < 4096 { + return nil, int(ret) + } + return unsafe.Pointer(ret), 0 + } + return sysMmap(addr, n, prot, flags, fd, off) +} + +func munmap(addr unsafe.Pointer, n uintptr) { + if _cgo_munmap != nil { + systemstack(func() { callCgoMunmap(addr, n) }) + return + } + sysMunmap(addr, n) +} + +// sysMmap calls the mmap system call. It is implemented in assembly. +func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) + +// callCgoMmap calls the mmap function in the runtime/cgo package +// using the GCC calling convention. It is implemented in assembly. +func callCgoMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) uintptr + +// sysMunmap calls the munmap system call. It is implemented in assembly. +func sysMunmap(addr unsafe.Pointer, n uintptr) + +// callCgoMunmap calls the munmap function in the runtime/cgo package +// using the GCC calling convention. It is implemented in assembly. +func callCgoMunmap(addr unsafe.Pointer, n uintptr) diff --git a/src/runtime/cgo_ppc64x.go b/src/runtime/cgo_ppc64x.go new file mode 100644 index 0000000..97b962e --- /dev/null +++ b/src/runtime/cgo_ppc64x.go @@ -0,0 +1,12 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ppc64 || ppc64le + +package runtime + +// crosscall_ppc64 calls into the runtime to set up the registers the +// Go runtime expects and so the symbol it calls needs to be exported +// for external linking to work. +//go:cgo_export_static _cgo_reginit diff --git a/src/runtime/cgo_sigaction.go b/src/runtime/cgo_sigaction.go new file mode 100644 index 0000000..a2e12f0 --- /dev/null +++ b/src/runtime/cgo_sigaction.go @@ -0,0 +1,92 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Support for sanitizers. See runtime/cgo/sigaction.go. + +//go:build (linux && amd64) || (freebsd && amd64) || (linux && arm64) || (linux && ppc64le) + +package runtime + +import "unsafe" + +// _cgo_sigaction is filled in by runtime/cgo when it is linked into the +// program, so it is only non-nil when using cgo. +//go:linkname _cgo_sigaction _cgo_sigaction +var _cgo_sigaction unsafe.Pointer + +//go:nosplit +//go:nowritebarrierrec +func sigaction(sig uint32, new, old *sigactiont) { + // racewalk.go avoids adding sanitizing instrumentation to package runtime, + // but we might be calling into instrumented C functions here, + // so we need the pointer parameters to be properly marked. + // + // Mark the input as having been written before the call + // and the output as read after. + if msanenabled && new != nil { + msanwrite(unsafe.Pointer(new), unsafe.Sizeof(*new)) + } + if asanenabled && new != nil { + asanwrite(unsafe.Pointer(new), unsafe.Sizeof(*new)) + } + if _cgo_sigaction == nil || inForkedChild { + sysSigaction(sig, new, old) + } else { + // We need to call _cgo_sigaction, which means we need a big enough stack + // for C. To complicate matters, we may be in libpreinit (before the + // runtime has been initialized) or in an asynchronous signal handler (with + // the current thread in transition between goroutines, or with the g0 + // system stack already in use). + + var ret int32 + + var g *g + if mainStarted { + g = getg() + } + sp := uintptr(unsafe.Pointer(&sig)) + switch { + case g == nil: + // No g: we're on a C stack or a signal stack. + ret = callCgoSigaction(uintptr(sig), new, old) + case sp < g.stack.lo || sp >= g.stack.hi: + // We're no longer on g's stack, so we must be handling a signal. It's + // possible that we interrupted the thread during a transition between g + // and g0, so we should stay on the current stack to avoid corrupting g0. + ret = callCgoSigaction(uintptr(sig), new, old) + default: + // We're running on g's stack, so either we're not in a signal handler or + // the signal handler has set the correct g. If we're on gsignal or g0, + // systemstack will make the call directly; otherwise, it will switch to + // g0 to ensure we have enough room to call a libc function. + // + // The function literal that we pass to systemstack is not nosplit, but + // that's ok: we'll be running on a fresh, clean system stack so the stack + // check will always succeed anyway. + systemstack(func() { + ret = callCgoSigaction(uintptr(sig), new, old) + }) + } + + const EINVAL = 22 + if ret == EINVAL { + // libc reserves certain signals — normally 32-33 — for pthreads, and + // returns EINVAL for sigaction calls on those signals. If we get EINVAL, + // fall back to making the syscall directly. + sysSigaction(sig, new, old) + } + } + + if msanenabled && old != nil { + msanread(unsafe.Pointer(old), unsafe.Sizeof(*old)) + } + if asanenabled && old != nil { + asanread(unsafe.Pointer(old), unsafe.Sizeof(*old)) + } +} + +// callCgoSigaction calls the sigaction function in the runtime/cgo package +// using the GCC calling convention. It is implemented in assembly. +//go:noescape +func callCgoSigaction(sig uintptr, new, old *sigactiont) int32 diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go new file mode 100644 index 0000000..a0c9560 --- /dev/null +++ b/src/runtime/cgocall.go @@ -0,0 +1,639 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Cgo call and callback support. +// +// To call into the C function f from Go, the cgo-generated code calls +// runtime.cgocall(_cgo_Cfunc_f, frame), where _cgo_Cfunc_f is a +// gcc-compiled function written by cgo. +// +// runtime.cgocall (below) calls entersyscall so as not to block +// other goroutines or the garbage collector, and then calls +// runtime.asmcgocall(_cgo_Cfunc_f, frame). +// +// runtime.asmcgocall (in asm_$GOARCH.s) switches to the m->g0 stack +// (assumed to be an operating system-allocated stack, so safe to run +// gcc-compiled code on) and calls _cgo_Cfunc_f(frame). +// +// _cgo_Cfunc_f invokes the actual C function f with arguments +// taken from the frame structure, records the results in the frame, +// and returns to runtime.asmcgocall. +// +// After it regains control, runtime.asmcgocall switches back to the +// original g (m->curg)'s stack and returns to runtime.cgocall. +// +// After it regains control, runtime.cgocall calls exitsyscall, which blocks +// until this m can run Go code without violating the $GOMAXPROCS limit, +// and then unlocks g from m. +// +// The above description skipped over the possibility of the gcc-compiled +// function f calling back into Go. If that happens, we continue down +// the rabbit hole during the execution of f. +// +// To make it possible for gcc-compiled C code to call a Go function p.GoF, +// cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't +// know about packages). The gcc-compiled C function f calls GoF. +// +// GoF initializes "frame", a structure containing all of its +// arguments and slots for p.GoF's results. It calls +// crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI. +// +// crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from +// the gcc function call ABI to the gc function call ABI. At this +// point we're in the Go runtime, but we're still running on m.g0's +// stack and outside the $GOMAXPROCS limit. crosscall2 calls +// runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI. +// (crosscall2's framesize argument is no longer used, but there's one +// case where SWIG calls crosscall2 directly and expects to pass this +// argument. See _cgo_panic.) +// +// runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack +// to the original g (m.curg)'s stack, on which it calls +// runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the +// stack switch, runtime.cgocallback saves the current SP as +// m.g0.sched.sp, so that any use of m.g0's stack during the execution +// of the callback will be done below the existing stack frames. +// Before overwriting m.g0.sched.sp, it pushes the old value on the +// m.g0 stack, so that it can be restored later. +// +// runtime.cgocallbackg (below) is now running on a real goroutine +// stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will +// block until the $GOMAXPROCS limit allows running this goroutine. +// Once exitsyscall has returned, it is safe to do things like call the memory +// allocator or invoke the Go callback function. runtime.cgocallbackg +// first defers a function to unwind m.g0.sched.sp, so that if p.GoF +// panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack +// and the m.curg stack will be unwound in lock step. +// Then it calls _cgoexp_GoF(frame). +// +// _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments +// from frame, calls p.GoF, writes the results back to frame, and +// returns. Now we start unwinding this whole process. +// +// runtime.cgocallbackg pops but does not execute the deferred +// function to unwind m.g0.sched.sp, calls runtime.entersyscall, and +// returns to runtime.cgocallback. +// +// After it regains control, runtime.cgocallback switches back to +// m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old +// m.g0.sched.sp value from the stack, and returns to crosscall2. +// +// crosscall2 restores the callee-save registers for gcc and returns +// to GoF, which unpacks any result values and returns to f. + +package runtime + +import ( + "internal/goarch" + "runtime/internal/atomic" + "runtime/internal/sys" + "unsafe" +) + +// Addresses collected in a cgo backtrace when crashing. +// Length must match arg.Max in x_cgo_callers in runtime/cgo/gcc_traceback.c. +type cgoCallers [32]uintptr + +// argset matches runtime/cgo/linux_syscall.c:argset_t +type argset struct { + args unsafe.Pointer + retval uintptr +} + +// wrapper for syscall package to call cgocall for libc (cgo) calls. +//go:linkname syscall_cgocaller syscall.cgocaller +//go:nosplit +//go:uintptrescapes +func syscall_cgocaller(fn unsafe.Pointer, args ...uintptr) uintptr { + as := argset{args: unsafe.Pointer(&args[0])} + cgocall(fn, unsafe.Pointer(&as)) + return as.retval +} + +var ncgocall uint64 // number of cgo calls in total for dead m + +// Call from Go to C. +// +// This must be nosplit because it's used for syscalls on some +// platforms. Syscalls may have untyped arguments on the stack, so +// it's not safe to grow or scan the stack. +// +//go:nosplit +func cgocall(fn, arg unsafe.Pointer) int32 { + if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" { + throw("cgocall unavailable") + } + + if fn == nil { + throw("cgocall nil") + } + + if raceenabled { + racereleasemerge(unsafe.Pointer(&racecgosync)) + } + + mp := getg().m + mp.ncgocall++ + mp.ncgo++ + + // Reset traceback. + mp.cgoCallers[0] = 0 + + // Announce we are entering a system call + // so that the scheduler knows to create another + // M to run goroutines while we are in the + // foreign code. + // + // The call to asmcgocall is guaranteed not to + // grow the stack and does not allocate memory, + // so it is safe to call while "in a system call", outside + // the $GOMAXPROCS accounting. + // + // fn may call back into Go code, in which case we'll exit the + // "system call", run the Go code (which may grow the stack), + // and then re-enter the "system call" reusing the PC and SP + // saved by entersyscall here. + entersyscall() + + // Tell asynchronous preemption that we're entering external + // code. We do this after entersyscall because this may block + // and cause an async preemption to fail, but at this point a + // sync preemption will succeed (though this is not a matter + // of correctness). + osPreemptExtEnter(mp) + + mp.incgo = true + errno := asmcgocall(fn, arg) + + // Update accounting before exitsyscall because exitsyscall may + // reschedule us on to a different M. + mp.incgo = false + mp.ncgo-- + + osPreemptExtExit(mp) + + exitsyscall() + + // Note that raceacquire must be called only after exitsyscall has + // wired this M to a P. + if raceenabled { + raceacquire(unsafe.Pointer(&racecgosync)) + } + + // From the garbage collector's perspective, time can move + // backwards in the sequence above. If there's a callback into + // Go code, GC will see this function at the call to + // asmcgocall. When the Go call later returns to C, the + // syscall PC/SP is rolled back and the GC sees this function + // back at the call to entersyscall. Normally, fn and arg + // would be live at entersyscall and dead at asmcgocall, so if + // time moved backwards, GC would see these arguments as dead + // and then live. Prevent these undead arguments from crashing + // GC by forcing them to stay live across this time warp. + KeepAlive(fn) + KeepAlive(arg) + KeepAlive(mp) + + return errno +} + +// Call from C back to Go. fn must point to an ABIInternal Go entry-point. +//go:nosplit +func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { + gp := getg() + if gp != gp.m.curg { + println("runtime: bad g in cgocallback") + exit(2) + } + + // The call from C is on gp.m's g0 stack, so we must ensure + // that we stay on that M. We have to do this before calling + // exitsyscall, since it would otherwise be free to move us to + // a different M. The call to unlockOSThread is in unwindm. + lockOSThread() + + checkm := gp.m + + // Save current syscall parameters, so m.syscall can be + // used again if callback decide to make syscall. + syscall := gp.m.syscall + + // entersyscall saves the caller's SP to allow the GC to trace the Go + // stack. However, since we're returning to an earlier stack frame and + // need to pair with the entersyscall() call made by cgocall, we must + // save syscall* and let reentersyscall restore them. + savedsp := unsafe.Pointer(gp.syscallsp) + savedpc := gp.syscallpc + exitsyscall() // coming out of cgo call + gp.m.incgo = false + + osPreemptExtExit(gp.m) + + cgocallbackg1(fn, frame, ctxt) // will call unlockOSThread + + // At this point unlockOSThread has been called. + // The following code must not change to a different m. + // This is enforced by checking incgo in the schedule function. + + gp.m.incgo = true + + if gp.m != checkm { + throw("m changed unexpectedly in cgocallbackg") + } + + osPreemptExtEnter(gp.m) + + // going back to cgo call + reentersyscall(savedpc, uintptr(savedsp)) + + gp.m.syscall = syscall +} + +func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { + gp := getg() + + // When we return, undo the call to lockOSThread in cgocallbackg. + // We must still stay on the same m. + defer unlockOSThread() + + if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 { + gp.m.needextram = false + systemstack(newextram) + } + + if ctxt != 0 { + s := append(gp.cgoCtxt, ctxt) + + // Now we need to set gp.cgoCtxt = s, but we could get + // a SIGPROF signal while manipulating the slice, and + // the SIGPROF handler could pick up gp.cgoCtxt while + // tracing up the stack. We need to ensure that the + // handler always sees a valid slice, so set the + // values in an order such that it always does. + p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) + atomicstorep(unsafe.Pointer(&p.array), unsafe.Pointer(&s[0])) + p.cap = cap(s) + p.len = len(s) + + defer func(gp *g) { + // Decrease the length of the slice by one, safely. + p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) + p.len-- + }(gp) + } + + if gp.m.ncgo == 0 { + // The C call to Go came from a thread not currently running + // any Go. In the case of -buildmode=c-archive or c-shared, + // this call may be coming in before package initialization + // is complete. Wait until it is. + <-main_init_done + } + + // Check whether the profiler needs to be turned on or off; this route to + // run Go code does not use runtime.execute, so bypasses the check there. + hz := sched.profilehz + if gp.m.profilehz != hz { + setThreadCPUProfiler(hz) + } + + // Add entry to defer stack in case of panic. + restore := true + defer unwindm(&restore) + + if raceenabled { + raceacquire(unsafe.Pointer(&racecgosync)) + } + + // Invoke callback. This function is generated by cmd/cgo and + // will unpack the argument frame and call the Go function. + var cb func(frame unsafe.Pointer) + cbFV := funcval{uintptr(fn)} + *(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV)) + cb(frame) + + if raceenabled { + racereleasemerge(unsafe.Pointer(&racecgosync)) + } + + // Do not unwind m->g0->sched.sp. + // Our caller, cgocallback, will do that. + restore = false +} + +func unwindm(restore *bool) { + if *restore { + // Restore sp saved by cgocallback during + // unwind of g's stack (see comment at top of file). + mp := acquirem() + sched := &mp.g0.sched + sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + alignUp(sys.MinFrameSize, sys.StackAlign))) + + // Do the accounting that cgocall will not have a chance to do + // during an unwind. + // + // In the case where a Go call originates from C, ncgo is 0 + // and there is no matching cgocall to end. + if mp.ncgo > 0 { + mp.incgo = false + mp.ncgo-- + osPreemptExtExit(mp) + } + + releasem(mp) + } +} + +// called from assembly +func badcgocallback() { + throw("misaligned stack in cgocallback") +} + +// called from (incomplete) assembly +func cgounimpl() { + throw("cgo not implemented") +} + +var racecgosync uint64 // represents possible synchronization in C code + +// Pointer checking for cgo code. + +// We want to detect all cases where a program that does not use +// unsafe makes a cgo call passing a Go pointer to memory that +// contains a Go pointer. Here a Go pointer is defined as a pointer +// to memory allocated by the Go runtime. Programs that use unsafe +// can evade this restriction easily, so we don't try to catch them. +// The cgo program will rewrite all possibly bad pointer arguments to +// call cgoCheckPointer, where we can catch cases of a Go pointer +// pointing to a Go pointer. + +// Complicating matters, taking the address of a slice or array +// element permits the C program to access all elements of the slice +// or array. In that case we will see a pointer to a single element, +// but we need to check the entire data structure. + +// The cgoCheckPointer call takes additional arguments indicating that +// it was called on an address expression. An additional argument of +// true means that it only needs to check a single element. An +// additional argument of a slice or array means that it needs to +// check the entire slice/array, but nothing else. Otherwise, the +// pointer could be anything, and we check the entire heap object, +// which is conservative but safe. + +// When and if we implement a moving garbage collector, +// cgoCheckPointer will pin the pointer for the duration of the cgo +// call. (This is necessary but not sufficient; the cgo program will +// also have to change to pin Go pointers that cannot point to Go +// pointers.) + +// cgoCheckPointer checks if the argument contains a Go pointer that +// points to a Go pointer, and panics if it does. +func cgoCheckPointer(ptr any, arg any) { + if debug.cgocheck == 0 { + return + } + + ep := efaceOf(&ptr) + t := ep._type + + top := true + if arg != nil && (t.kind&kindMask == kindPtr || t.kind&kindMask == kindUnsafePointer) { + p := ep.data + if t.kind&kindDirectIface == 0 { + p = *(*unsafe.Pointer)(p) + } + if p == nil || !cgoIsGoPointer(p) { + return + } + aep := efaceOf(&arg) + switch aep._type.kind & kindMask { + case kindBool: + if t.kind&kindMask == kindUnsafePointer { + // We don't know the type of the element. + break + } + pt := (*ptrtype)(unsafe.Pointer(t)) + cgoCheckArg(pt.elem, p, true, false, cgoCheckPointerFail) + return + case kindSlice: + // Check the slice rather than the pointer. + ep = aep + t = ep._type + case kindArray: + // Check the array rather than the pointer. + // Pass top as false since we have a pointer + // to the array. + ep = aep + t = ep._type + top = false + default: + throw("can't happen") + } + } + + cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, top, cgoCheckPointerFail) +} + +const cgoCheckPointerFail = "cgo argument has Go pointer to Go pointer" +const cgoResultFail = "cgo result has Go pointer" + +// cgoCheckArg is the real work of cgoCheckPointer. The argument p +// is either a pointer to the value (of type t), or the value itself, +// depending on indir. The top parameter is whether we are at the top +// level, where Go pointers are allowed. +func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { + if t.ptrdata == 0 || p == nil { + // If the type has no pointers there is nothing to do. + return + } + + switch t.kind & kindMask { + default: + throw("can't happen") + case kindArray: + at := (*arraytype)(unsafe.Pointer(t)) + if !indir { + if at.len != 1 { + throw("can't happen") + } + cgoCheckArg(at.elem, p, at.elem.kind&kindDirectIface == 0, top, msg) + return + } + for i := uintptr(0); i < at.len; i++ { + cgoCheckArg(at.elem, p, true, top, msg) + p = add(p, at.elem.size) + } + case kindChan, kindMap: + // These types contain internal pointers that will + // always be allocated in the Go heap. It's never OK + // to pass them to C. + panic(errorString(msg)) + case kindFunc: + if indir { + p = *(*unsafe.Pointer)(p) + } + if !cgoIsGoPointer(p) { + return + } + panic(errorString(msg)) + case kindInterface: + it := *(**_type)(p) + if it == nil { + return + } + // A type known at compile time is OK since it's + // constant. A type not known at compile time will be + // in the heap and will not be OK. + if inheap(uintptr(unsafe.Pointer(it))) { + panic(errorString(msg)) + } + p = *(*unsafe.Pointer)(add(p, goarch.PtrSize)) + if !cgoIsGoPointer(p) { + return + } + if !top { + panic(errorString(msg)) + } + cgoCheckArg(it, p, it.kind&kindDirectIface == 0, false, msg) + case kindSlice: + st := (*slicetype)(unsafe.Pointer(t)) + s := (*slice)(p) + p = s.array + if p == nil || !cgoIsGoPointer(p) { + return + } + if !top { + panic(errorString(msg)) + } + if st.elem.ptrdata == 0 { + return + } + for i := 0; i < s.cap; i++ { + cgoCheckArg(st.elem, p, true, false, msg) + p = add(p, st.elem.size) + } + case kindString: + ss := (*stringStruct)(p) + if !cgoIsGoPointer(ss.str) { + return + } + if !top { + panic(errorString(msg)) + } + case kindStruct: + st := (*structtype)(unsafe.Pointer(t)) + if !indir { + if len(st.fields) != 1 { + throw("can't happen") + } + cgoCheckArg(st.fields[0].typ, p, st.fields[0].typ.kind&kindDirectIface == 0, top, msg) + return + } + for _, f := range st.fields { + if f.typ.ptrdata == 0 { + continue + } + cgoCheckArg(f.typ, add(p, f.offset()), true, top, msg) + } + case kindPtr, kindUnsafePointer: + if indir { + p = *(*unsafe.Pointer)(p) + if p == nil { + return + } + } + + if !cgoIsGoPointer(p) { + return + } + if !top { + panic(errorString(msg)) + } + + cgoCheckUnknownPointer(p, msg) + } +} + +// cgoCheckUnknownPointer is called for an arbitrary pointer into Go +// memory. It checks whether that Go memory contains any other +// pointer into Go memory. If it does, we panic. +// The return values are unused but useful to see in panic tracebacks. +func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) { + if inheap(uintptr(p)) { + b, span, _ := findObject(uintptr(p), 0, 0) + base = b + if base == 0 { + return + } + hbits := heapBitsForAddr(base) + n := span.elemsize + for i = uintptr(0); i < n; i += goarch.PtrSize { + if !hbits.morePointers() { + // No more possible pointers. + break + } + if hbits.isPointer() && cgoIsGoPointer(*(*unsafe.Pointer)(unsafe.Pointer(base + i))) { + panic(errorString(msg)) + } + hbits = hbits.next() + } + + return + } + + for _, datap := range activeModules() { + if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) { + // We have no way to know the size of the object. + // We have to assume that it might contain a pointer. + panic(errorString(msg)) + } + // In the text or noptr sections, we know that the + // pointer does not point to a Go pointer. + } + + return +} + +// cgoIsGoPointer reports whether the pointer is a Go pointer--a +// pointer to Go memory. We only care about Go memory that might +// contain pointers. +//go:nosplit +//go:nowritebarrierrec +func cgoIsGoPointer(p unsafe.Pointer) bool { + if p == nil { + return false + } + + if inHeapOrStack(uintptr(p)) { + return true + } + + for _, datap := range activeModules() { + if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) { + return true + } + } + + return false +} + +// cgoInRange reports whether p is between start and end. +//go:nosplit +//go:nowritebarrierrec +func cgoInRange(p unsafe.Pointer, start, end uintptr) bool { + return start <= uintptr(p) && uintptr(p) < end +} + +// cgoCheckResult is called to check the result parameter of an +// exported Go function. It panics if the result is or contains a Go +// pointer. +func cgoCheckResult(val any) { + if debug.cgocheck == 0 { + return + } + + ep := efaceOf(&val) + t := ep._type + cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, false, cgoResultFail) +} diff --git a/src/runtime/cgocallback.go b/src/runtime/cgocallback.go new file mode 100644 index 0000000..59953f1 --- /dev/null +++ b/src/runtime/cgocallback.go @@ -0,0 +1,13 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// These functions are called from C code via cgo/callbacks.go. + +// Panic. + +func _cgo_panic_internal(p *byte) { + panic(gostringnocopy(p)) +} diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go new file mode 100644 index 0000000..3acbadf --- /dev/null +++ b/src/runtime/cgocheck.go @@ -0,0 +1,263 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code to check that pointer writes follow the cgo rules. +// These functions are invoked via the write barrier when debug.cgocheck > 1. + +package runtime + +import ( + "internal/goarch" + "unsafe" +) + +const cgoWriteBarrierFail = "Go pointer stored into non-Go memory" + +// cgoCheckWriteBarrier is called whenever a pointer is stored into memory. +// It throws if the program is storing a Go pointer into non-Go memory. +// +// This is called from the write barrier, so its entire call tree must +// be nosplit. +// +//go:nosplit +//go:nowritebarrier +func cgoCheckWriteBarrier(dst *uintptr, src uintptr) { + if !cgoIsGoPointer(unsafe.Pointer(src)) { + return + } + if cgoIsGoPointer(unsafe.Pointer(dst)) { + return + } + + // If we are running on the system stack then dst might be an + // address on the stack, which is OK. + g := getg() + if g == g.m.g0 || g == g.m.gsignal { + return + } + + // Allocating memory can write to various mfixalloc structs + // that look like they are non-Go memory. + if g.m.mallocing != 0 { + return + } + + // It's OK if writing to memory allocated by persistentalloc. + // Do this check last because it is more expensive and rarely true. + // If it is false the expense doesn't matter since we are crashing. + if inPersistentAlloc(uintptr(unsafe.Pointer(dst))) { + return + } + + systemstack(func() { + println("write of Go pointer", hex(src), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst)))) + throw(cgoWriteBarrierFail) + }) +} + +// cgoCheckMemmove is called when moving a block of memory. +// dst and src point off bytes into the value to copy. +// size is the number of bytes to copy. +// It throws if the program is copying a block that contains a Go pointer +// into non-Go memory. +//go:nosplit +//go:nowritebarrier +func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer, off, size uintptr) { + if typ.ptrdata == 0 { + return + } + if !cgoIsGoPointer(src) { + return + } + if cgoIsGoPointer(dst) { + return + } + cgoCheckTypedBlock(typ, src, off, size) +} + +// cgoCheckSliceCopy is called when copying n elements of a slice. +// src and dst are pointers to the first element of the slice. +// typ is the element type of the slice. +// It throws if the program is copying slice elements that contain Go pointers +// into non-Go memory. +//go:nosplit +//go:nowritebarrier +func cgoCheckSliceCopy(typ *_type, dst, src unsafe.Pointer, n int) { + if typ.ptrdata == 0 { + return + } + if !cgoIsGoPointer(src) { + return + } + if cgoIsGoPointer(dst) { + return + } + p := src + for i := 0; i < n; i++ { + cgoCheckTypedBlock(typ, p, 0, typ.size) + p = add(p, typ.size) + } +} + +// cgoCheckTypedBlock checks the block of memory at src, for up to size bytes, +// and throws if it finds a Go pointer. The type of the memory is typ, +// and src is off bytes into that type. +//go:nosplit +//go:nowritebarrier +func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { + // Anything past typ.ptrdata is not a pointer. + if typ.ptrdata <= off { + return + } + if ptrdataSize := typ.ptrdata - off; size > ptrdataSize { + size = ptrdataSize + } + + if typ.kind&kindGCProg == 0 { + cgoCheckBits(src, typ.gcdata, off, size) + return + } + + // The type has a GC program. Try to find GC bits somewhere else. + for _, datap := range activeModules() { + if cgoInRange(src, datap.data, datap.edata) { + doff := uintptr(src) - datap.data + cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size) + return + } + if cgoInRange(src, datap.bss, datap.ebss) { + boff := uintptr(src) - datap.bss + cgoCheckBits(add(src, -boff), datap.gcbssmask.bytedata, off+boff, size) + return + } + } + + s := spanOfUnchecked(uintptr(src)) + if s.state.get() == mSpanManual { + // There are no heap bits for value stored on the stack. + // For a channel receive src might be on the stack of some + // other goroutine, so we can't unwind the stack even if + // we wanted to. + // We can't expand the GC program without extra storage + // space we can't easily get. + // Fortunately we have the type information. + systemstack(func() { + cgoCheckUsingType(typ, src, off, size) + }) + return + } + + // src must be in the regular heap. + + hbits := heapBitsForAddr(uintptr(src)) + for i := uintptr(0); i < off+size; i += goarch.PtrSize { + bits := hbits.bits() + if i >= off && bits&bitPointer != 0 { + v := *(*unsafe.Pointer)(add(src, i)) + if cgoIsGoPointer(v) { + throw(cgoWriteBarrierFail) + } + } + hbits = hbits.next() + } +} + +// cgoCheckBits checks the block of memory at src, for up to size +// bytes, and throws if it finds a Go pointer. The gcbits mark each +// pointer value. The src pointer is off bytes into the gcbits. +//go:nosplit +//go:nowritebarrier +func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) { + skipMask := off / goarch.PtrSize / 8 + skipBytes := skipMask * goarch.PtrSize * 8 + ptrmask := addb(gcbits, skipMask) + src = add(src, skipBytes) + off -= skipBytes + size += off + var bits uint32 + for i := uintptr(0); i < size; i += goarch.PtrSize { + if i&(goarch.PtrSize*8-1) == 0 { + bits = uint32(*ptrmask) + ptrmask = addb(ptrmask, 1) + } else { + bits >>= 1 + } + if off > 0 { + off -= goarch.PtrSize + } else { + if bits&1 != 0 { + v := *(*unsafe.Pointer)(add(src, i)) + if cgoIsGoPointer(v) { + throw(cgoWriteBarrierFail) + } + } + } + } +} + +// cgoCheckUsingType is like cgoCheckTypedBlock, but is a last ditch +// fall back to look for pointers in src using the type information. +// We only use this when looking at a value on the stack when the type +// uses a GC program, because otherwise it's more efficient to use the +// GC bits. This is called on the system stack. +//go:nowritebarrier +//go:systemstack +func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) { + if typ.ptrdata == 0 { + return + } + + // Anything past typ.ptrdata is not a pointer. + if typ.ptrdata <= off { + return + } + if ptrdataSize := typ.ptrdata - off; size > ptrdataSize { + size = ptrdataSize + } + + if typ.kind&kindGCProg == 0 { + cgoCheckBits(src, typ.gcdata, off, size) + return + } + switch typ.kind & kindMask { + default: + throw("can't happen") + case kindArray: + at := (*arraytype)(unsafe.Pointer(typ)) + for i := uintptr(0); i < at.len; i++ { + if off < at.elem.size { + cgoCheckUsingType(at.elem, src, off, size) + } + src = add(src, at.elem.size) + skipped := off + if skipped > at.elem.size { + skipped = at.elem.size + } + checked := at.elem.size - skipped + off -= skipped + if size <= checked { + return + } + size -= checked + } + case kindStruct: + st := (*structtype)(unsafe.Pointer(typ)) + for _, f := range st.fields { + if off < f.typ.size { + cgoCheckUsingType(f.typ, src, off, size) + } + src = add(src, f.typ.size) + skipped := off + if skipped > f.typ.size { + skipped = f.typ.size + } + checked := f.typ.size - skipped + off -= skipped + if size <= checked { + return + } + size -= checked + } + } +} |