summaryrefslogtreecommitdiffstats
path: root/src/runtime/race_ppc64le.s
blob: 8961254ea6efb0f29829c9fa9889c0121601aa8f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build race

#include "go_asm.h"
#include "go_tls.h"
#include "funcdata.h"
#include "textflag.h"
#include "asm_ppc64x.h"

// The following functions allow calling the clang-compiled race runtime directly
// from Go code without going all the way through cgo.
// First, it's much faster (up to 50% speedup for real Go programs).
// Second, it eliminates race-related special cases from cgocall and scheduler.
// Third, in long-term it will allow to remove cyclic runtime/race dependency on cmd/go.

// A brief recap of the ppc64le calling convention.
// Arguments are passed in R3, R4, R5 ...
// SP must be 16-byte aligned.

// Note that for ppc64x, LLVM follows the standard ABI and
// expects arguments in registers, so these functions move
// the arguments from storage to the registers expected
// by the ABI.

// When calling from Go to Clang tsan code:
// R3 is the 1st argument and is usually the ThreadState*
// R4-? are the 2nd, 3rd, 4th, etc. arguments

// When calling racecalladdr:
// R8 is the call target address

// The race ctx is passed in R3 and loaded in
// racecalladdr.
//
// The sequence used to get the race ctx:
//    MOVD    runtime·tls_g(SB), R10	// offset to TLS
//    MOVD    0(R13)(R10*1), g		// R13=TLS for this thread, g = R30
//    MOVD    g_racectx(g), R3		// racectx == ThreadState

// func runtime·RaceRead(addr uintptr)
// Called from instrumented Go code
TEXT	runtime·raceread(SB), NOSPLIT, $0-8
	MOVD	addr+0(FP), R4
	MOVD	LR, R5 // caller of this?
	// void __tsan_read(ThreadState *thr, void *addr, void *pc);
	MOVD	$__tsan_read(SB), R8
	BR	racecalladdr<>(SB)

TEXT    runtime·RaceRead(SB), NOSPLIT, $0-8
	BR	runtime·raceread(SB)

// void runtime·racereadpc(void *addr, void *callpc, void *pc)
TEXT	runtime·racereadpc(SB), NOSPLIT, $0-24
	MOVD	addr+0(FP), R4
	MOVD	callpc+8(FP), R5
	MOVD	pc+16(FP), R6
	// void __tsan_read_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
	MOVD	$__tsan_read_pc(SB), R8
	BR	racecalladdr<>(SB)

// func runtime·RaceWrite(addr uintptr)
// Called from instrumented Go code
TEXT	runtime·racewrite(SB), NOSPLIT, $0-8
	MOVD	addr+0(FP), R4
	MOVD	LR, R5 // caller has set LR via BL inst
	// void __tsan_write(ThreadState *thr, void *addr, void *pc);
	MOVD	$__tsan_write(SB), R8
	BR	racecalladdr<>(SB)

TEXT    runtime·RaceWrite(SB), NOSPLIT, $0-8
	JMP	runtime·racewrite(SB)

// void runtime·racewritepc(void *addr, void *callpc, void *pc)
TEXT	runtime·racewritepc(SB), NOSPLIT, $0-24
	MOVD	addr+0(FP), R4
	MOVD	callpc+8(FP), R5
	MOVD	pc+16(FP), R6
	// void __tsan_write_pc(ThreadState *thr, void *addr, void *callpc, void *pc);
	MOVD	$__tsan_write_pc(SB), R8
	BR	racecalladdr<>(SB)

// func runtime·RaceReadRange(addr, size uintptr)
// Called from instrumented Go code.
TEXT	runtime·racereadrange(SB), NOSPLIT, $0-16
	MOVD	addr+0(FP), R4
	MOVD	size+8(FP), R5
	MOVD	LR, R6
	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
	MOVD	$__tsan_read_range(SB), R8
	BR	racecalladdr<>(SB)

// void runtime·racereadrangepc1(void *addr, uintptr sz, void *pc)
TEXT	runtime·racereadrangepc1(SB), NOSPLIT, $0-24
	MOVD    addr+0(FP), R4
	MOVD    size+8(FP), R5
	MOVD    pc+16(FP), R6
	ADD	$4, R6		// tsan wants return addr
	// void __tsan_read_range(ThreadState *thr, void *addr, uintptr size, void *pc);
	MOVD    $__tsan_read_range(SB), R8
	BR	racecalladdr<>(SB)

TEXT    runtime·RaceReadRange(SB), NOSPLIT, $0-16
	BR	runtime·racereadrange(SB)

// func runtime·RaceWriteRange(addr, size uintptr)
// Called from instrumented Go code.
TEXT	runtime·racewriterange(SB), NOSPLIT, $0-16
	MOVD	addr+0(FP), R4
	MOVD	size+8(FP), R5
	MOVD	LR, R6
	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
	MOVD	$__tsan_write_range(SB), R8
	BR	racecalladdr<>(SB)

TEXT    runtime·RaceWriteRange(SB), NOSPLIT, $0-16
	BR	runtime·racewriterange(SB)

// void runtime·racewriterangepc1(void *addr, uintptr sz, void *pc)
// Called from instrumented Go code
TEXT	runtime·racewriterangepc1(SB), NOSPLIT, $0-24
	MOVD	addr+0(FP), R4
	MOVD	size+8(FP), R5
	MOVD	pc+16(FP), R6
	ADD	$4, R6			// add 4 to inst offset?
	// void __tsan_write_range(ThreadState *thr, void *addr, uintptr size, void *pc);
	MOVD	$__tsan_write_range(SB), R8
	BR	racecalladdr<>(SB)

// Call a __tsan function from Go code.
// R8 = tsan function address
// R3 = *ThreadState a.k.a. g_racectx from g
// R4 = addr passed to __tsan function
//
// Otherwise, setup goroutine context and invoke racecall. Other arguments already set.
TEXT	racecalladdr<>(SB), NOSPLIT, $0-0
	MOVD    runtime·tls_g(SB), R10
	MOVD	0(R13)(R10*1), g
	MOVD	g_racectx(g), R3	// goroutine context
	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
	MOVD	runtime·racearenastart(SB), R9
	CMP	R4, R9
	BLT	data
	MOVD	runtime·racearenaend(SB), R9
	CMP	R4, R9
	BLT	call
data:
	MOVD	runtime·racedatastart(SB), R9
	CMP	R4, R9
	BLT	ret
	MOVD	runtime·racedataend(SB), R9
	CMP	R4, R9
	BGT	ret
call:
	// Careful!! racecall will save LR on its
	// stack, which is OK as long as racecalladdr
	// doesn't change in a way that generates a stack.
	// racecall should return to the caller of
	// recalladdr.
	BR	racecall<>(SB)
ret:
	RET

// func runtime·racefuncenterfp()
// Called from instrumented Go code.
// Like racefuncenter but doesn't pass an arg, uses the caller pc
// from the first slot on the stack.
TEXT	runtime·racefuncenterfp(SB), NOSPLIT, $0-0
	MOVD	0(R1), R8
	BR	racefuncenter<>(SB)

// func runtime·racefuncenter(pc uintptr)
// Called from instrumented Go code.
// Not used now since gc/racewalk.go doesn't pass the
// correct caller pc and racefuncenterfp can do it.
TEXT	runtime·racefuncenter(SB), NOSPLIT, $0-8
	MOVD	callpc+0(FP), R8
	BR	racefuncenter<>(SB)

// Common code for racefuncenter/racefuncenterfp
// R11 = caller's return address
TEXT	racefuncenter<>(SB), NOSPLIT, $0-0
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g
	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
	MOVD	R8, R4			// caller pc set by caller in R8
	// void __tsan_func_enter(ThreadState *thr, void *pc);
	MOVD	$__tsan_func_enter(SB), R8
	BR	racecall<>(SB)
	RET

// func runtime·racefuncexit()
// Called from Go instrumented code.
TEXT	runtime·racefuncexit(SB), NOSPLIT, $0-0
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g
	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
	// void __tsan_func_exit(ThreadState *thr);
	MOVD	$__tsan_func_exit(SB), R8
	BR	racecall<>(SB)

// Atomic operations for sync/atomic package.
// Some use the __tsan versions instead
// R6 = addr of arguments passed to this function
// R3, R4, R5 set in racecallatomic

// Load atomic in tsan
TEXT	syncatomic·LoadInt32(SB), NOSPLIT, $0-12
	GO_ARGS
	// void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic32_load(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)
	RET

TEXT	syncatomic·LoadInt64(SB), NOSPLIT, $0-16
	GO_ARGS
	// void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic64_load(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)
	RET

TEXT	syncatomic·LoadUint32(SB), NOSPLIT, $0-12
	GO_ARGS
	BR	syncatomic·LoadInt32(SB)

TEXT	syncatomic·LoadUint64(SB), NOSPLIT, $0-16
	GO_ARGS
	BR	syncatomic·LoadInt64(SB)

TEXT	syncatomic·LoadUintptr(SB), NOSPLIT, $0-16
	GO_ARGS
	BR	syncatomic·LoadInt64(SB)

TEXT	syncatomic·LoadPointer(SB), NOSPLIT, $0-16
	GO_ARGS
	BR	syncatomic·LoadInt64(SB)

// Store atomic in tsan
TEXT	syncatomic·StoreInt32(SB), NOSPLIT, $0-12
	GO_ARGS
	// void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic32_store(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·StoreInt64(SB), NOSPLIT, $0-16
	GO_ARGS
	// void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic64_store(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·StoreUint32(SB), NOSPLIT, $0-12
	GO_ARGS
	BR	syncatomic·StoreInt32(SB)

TEXT	syncatomic·StoreUint64(SB), NOSPLIT, $0-16
	GO_ARGS
	BR	syncatomic·StoreInt64(SB)

TEXT	syncatomic·StoreUintptr(SB), NOSPLIT, $0-16
	GO_ARGS
	BR	syncatomic·StoreInt64(SB)

// Swap in tsan
TEXT	syncatomic·SwapInt32(SB), NOSPLIT, $0-20
	GO_ARGS
	// void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic32_exchange(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·SwapInt64(SB), NOSPLIT, $0-24
	GO_ARGS
	// void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a)
	MOVD	$__tsan_go_atomic64_exchange(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·SwapUint32(SB), NOSPLIT, $0-20
	GO_ARGS
	BR	syncatomic·SwapInt32(SB)

TEXT	syncatomic·SwapUint64(SB), NOSPLIT, $0-24
	GO_ARGS
	BR	syncatomic·SwapInt64(SB)

TEXT	syncatomic·SwapUintptr(SB), NOSPLIT, $0-24
	GO_ARGS
	BR	syncatomic·SwapInt64(SB)

// Add atomic in tsan
TEXT	syncatomic·AddInt32(SB), NOSPLIT, $0-20
	GO_ARGS
	// void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic32_fetch_add(SB), R8
	ADD	$64, R1, R6	// addr of caller's 1st arg
	BL	racecallatomic<>(SB)
	// The tsan fetch_add result is not as expected by Go,
	// so the 'add' must be added to the result.
	MOVW	add+8(FP), R3	// The tsa fetch_add does not return the
	MOVW	ret+16(FP), R4	// result as expected by go, so fix it.
	ADD	R3, R4, R3
	MOVW	R3, ret+16(FP)
	RET

TEXT	syncatomic·AddInt64(SB), NOSPLIT, $0-24
	GO_ARGS
	// void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a);
	MOVD	$__tsan_go_atomic64_fetch_add(SB), R8
	ADD	$64, R1, R6	// addr of caller's 1st arg
	BL	racecallatomic<>(SB)
	// The tsan fetch_add result is not as expected by Go,
	// so the 'add' must be added to the result.
	MOVD	add+8(FP), R3
	MOVD	ret+16(FP), R4
	ADD	R3, R4, R3
	MOVD	R3, ret+16(FP)
	RET

TEXT	syncatomic·AddUint32(SB), NOSPLIT, $0-20
	GO_ARGS
	BR	syncatomic·AddInt32(SB)

TEXT	syncatomic·AddUint64(SB), NOSPLIT, $0-24
	GO_ARGS
	BR	syncatomic·AddInt64(SB)

TEXT	syncatomic·AddUintptr(SB), NOSPLIT, $0-24
	GO_ARGS
	BR	syncatomic·AddInt64(SB)

// CompareAndSwap in tsan
TEXT	syncatomic·CompareAndSwapInt32(SB), NOSPLIT, $0-17
	GO_ARGS
	// void __tsan_go_atomic32_compare_exchange(
	//   ThreadState *thr, uptr cpc, uptr pc, u8 *a)
	MOVD	$__tsan_go_atomic32_compare_exchange(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·CompareAndSwapInt64(SB), NOSPLIT, $0-25
	GO_ARGS
	// void __tsan_go_atomic32_compare_exchange(
	//   ThreadState *thr, uptr cpc, uptr pc, u8 *a)
	MOVD	$__tsan_go_atomic64_compare_exchange(SB), R8
	ADD	$32, R1, R6	// addr of caller's 1st arg
	BR	racecallatomic<>(SB)

TEXT	syncatomic·CompareAndSwapUint32(SB), NOSPLIT, $0-17
	GO_ARGS
	BR	syncatomic·CompareAndSwapInt32(SB)

TEXT	syncatomic·CompareAndSwapUint64(SB), NOSPLIT, $0-25
	GO_ARGS
	BR	syncatomic·CompareAndSwapInt64(SB)

TEXT	syncatomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-25
	GO_ARGS
	BR	syncatomic·CompareAndSwapInt64(SB)

// Common function used to call tsan's atomic functions
// R3 = *ThreadState
// R4 = TODO: What's this supposed to be?
// R5 = caller pc
// R6 = addr of incoming arg list
// R8 contains addr of target function.
TEXT	racecallatomic<>(SB), NOSPLIT, $0-0
	// Trigger SIGSEGV early if address passed to atomic function is bad.
	MOVD	(R6), R7	// 1st arg is addr
	MOVD	(R7), R9	// segv here if addr is bad
	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
	MOVD	runtime·racearenastart(SB), R9
	CMP	R7, R9
	BLT	racecallatomic_data
	MOVD	runtime·racearenaend(SB), R9
	CMP	R7, R9
	BLT	racecallatomic_ok
racecallatomic_data:
	MOVD	runtime·racedatastart(SB), R9
	CMP	R7, R9
	BLT	racecallatomic_ignore
	MOVD	runtime·racedataend(SB), R9
	CMP	R7, R9
	BGE	racecallatomic_ignore
racecallatomic_ok:
	// Addr is within the good range, call the atomic function.
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g
	MOVD    g_racectx(g), R3        // goroutine racectx aka *ThreadState
	MOVD	R8, R5			// pc is the function called
	MOVD	(R1), R4		// caller pc from stack
	BL	racecall<>(SB)		// BL needed to maintain stack consistency
	RET				//
racecallatomic_ignore:
	// Addr is outside the good range.
	// Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.
	// An attempt to synchronize on the address would cause crash.
	MOVD	R8, R15	// save the original function
	MOVD	R6, R17 // save the original arg list addr
	MOVD	$__tsan_go_ignore_sync_begin(SB), R8 // func addr to call
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g
	MOVD    g_racectx(g), R3        // goroutine context
	BL	racecall<>(SB)
	MOVD	R15, R8	// restore the original function
	MOVD	R17, R6 // restore arg list addr
	// Call the atomic function.
	// racecall will call LLVM race code which might clobber r30 (g)
	MOVD	runtime·tls_g(SB), R10
	MOVD	0(R13)(R10*1), g

	MOVD	g_racectx(g), R3
	MOVD	R8, R4		// pc being called same TODO as above
	MOVD	(R1), R5	// caller pc from latest LR
	BL	racecall<>(SB)
	// Call __tsan_go_ignore_sync_end.
	MOVD	$__tsan_go_ignore_sync_end(SB), R8
	MOVD	g_racectx(g), R3	// goroutine context g should sitll be good?
	BL	racecall<>(SB)
	RET

// void runtime·racecall(void(*f)(...), ...)
// Calls C function f from race runtime and passes up to 4 arguments to it.
// The arguments are never heap-object-preserving pointers, so we pretend there are no arguments.
TEXT	runtime·racecall(SB), NOSPLIT, $0-0
	MOVD	fn+0(FP), R8
	MOVD	arg0+8(FP), R3
	MOVD	arg1+16(FP), R4
	MOVD	arg2+24(FP), R5
	MOVD	arg3+32(FP), R6
	JMP	racecall<>(SB)

// Finds g0 and sets its stack
// Arguments were loaded for call from Go to C
TEXT	racecall<>(SB), NOSPLIT, $0-0
	// Set the LR slot for the ppc64 ABI
	MOVD	LR, R10
	MOVD	R10, 0(R1)	// Go expectation
	MOVD	R10, 16(R1)	// C ABI
	// Get info from the current goroutine
	MOVD    runtime·tls_g(SB), R10	// g offset in TLS
	MOVD    0(R13)(R10*1), g	// R13 = current TLS
	MOVD	g_m(g), R7		// m for g
	MOVD	R1, R16			// callee-saved, preserved across C call
	MOVD	m_g0(R7), R10		// g0 for m
	CMP	R10, g			// same g0?
	BEQ	call			// already on g0
	MOVD	(g_sched+gobuf_sp)(R10), R1 // switch R1
call:
	MOVD	R8, CTR			// R8 = caller addr
	MOVD	R8, R12			// expected by PPC64 ABI
	BL	(CTR)
	XOR     R0, R0			// clear R0 on return from Clang
	MOVD	R16, R1			// restore R1; R16 nonvol in Clang
	MOVD    runtime·tls_g(SB), R10	// find correct g
	MOVD    0(R13)(R10*1), g
	MOVD	16(R1), R10		// LR was saved away, restore for return
	MOVD	R10, LR
	RET

// C->Go callback thunk that allows to call runtime·racesymbolize from C code.
// Direct Go->C race call has only switched SP, finish g->g0 switch by setting correct g.
// The overall effect of Go->C->Go call chain is similar to that of mcall.
// RARG0 contains command code. RARG1 contains command-specific context.
// See racecallback for command codes.
TEXT	runtime·racecallbackthunk(SB), NOSPLIT, $-8
	// Handle command raceGetProcCmd (0) here.
	// First, code below assumes that we are on curg, while raceGetProcCmd
	// can be executed on g0. Second, it is called frequently, so will
	// benefit from this fast path.
	XOR	R0, R0		// clear R0 since we came from C code
	CMP	R3, $0
	BNE	rest
	// g0 TODO: Don't modify g here since R30 is nonvolatile
	MOVD	g, R9
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g
	MOVD	g_m(g), R3
	MOVD	m_p(R3), R3
	MOVD	p_raceprocctx(R3), R3
	MOVD	R3, (R4)
	MOVD	R9, g		// restore R30 ??
	RET

	// This is all similar to what cgo does
	// Save registers according to the ppc64 ABI
rest:
	MOVD	LR, R10	// save link register
	MOVD	R10, 16(R1)
	MOVW	CR, R10
	MOVW	R10, 8(R1)
	MOVDU   R1, -336(R1) // Allocate frame needed for outargs and register save area

	MOVD    R14, 328(R1)
	MOVD    R15, 48(R1)
	MOVD    R16, 56(R1)
	MOVD    R17, 64(R1)
	MOVD    R18, 72(R1)
	MOVD    R19, 80(R1)
	MOVD    R20, 88(R1)
	MOVD    R21, 96(R1)
	MOVD    R22, 104(R1)
	MOVD    R23, 112(R1)
	MOVD    R24, 120(R1)
	MOVD    R25, 128(R1)
	MOVD    R26, 136(R1)
	MOVD    R27, 144(R1)
	MOVD    R28, 152(R1)
	MOVD    R29, 160(R1)
	MOVD    g, 168(R1) // R30
	MOVD    R31, 176(R1)
	FMOVD   F14, 184(R1)
	FMOVD   F15, 192(R1)
	FMOVD   F16, 200(R1)
	FMOVD   F17, 208(R1)
	FMOVD   F18, 216(R1)
	FMOVD   F19, 224(R1)
	FMOVD   F20, 232(R1)
	FMOVD   F21, 240(R1)
	FMOVD   F22, 248(R1)
	FMOVD   F23, 256(R1)
	FMOVD   F24, 264(R1)
	FMOVD   F25, 272(R1)
	FMOVD   F26, 280(R1)
	FMOVD   F27, 288(R1)
	FMOVD   F28, 296(R1)
	FMOVD   F29, 304(R1)
	FMOVD   F30, 312(R1)
	FMOVD   F31, 320(R1)

	MOVD	R3, FIXED_FRAME+0(R1)
	MOVD	R4, FIXED_FRAME+8(R1)

	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g

	MOVD	g_m(g), R7
	MOVD	m_g0(R7), R8
	CMP	g, R8
	BEQ	noswitch

	MOVD	R8, g // set g = m-> g0

	BL	runtime·racecallback(SB)

	// All registers are clobbered after Go code, reload.
	MOVD    runtime·tls_g(SB), R10
	MOVD    0(R13)(R10*1), g

	MOVD	g_m(g), R7
	MOVD	m_curg(R7), g // restore g = m->curg

ret:
	MOVD    328(R1), R14
	MOVD    48(R1), R15
	MOVD    56(R1), R16
	MOVD    64(R1), R17
	MOVD    72(R1), R18
	MOVD    80(R1), R19
	MOVD    88(R1), R20
	MOVD    96(R1), R21
	MOVD    104(R1), R22
	MOVD    112(R1), R23
	MOVD    120(R1), R24
	MOVD    128(R1), R25
	MOVD    136(R1), R26
	MOVD    144(R1), R27
	MOVD    152(R1), R28
	MOVD    160(R1), R29
	MOVD    168(R1), g // R30
	MOVD    176(R1), R31
	FMOVD   184(R1), F14
	FMOVD   192(R1), F15
	FMOVD   200(R1), F16
	FMOVD   208(R1), F17
	FMOVD   216(R1), F18
	FMOVD   224(R1), F19
	FMOVD   232(R1), F20
	FMOVD   240(R1), F21
	FMOVD   248(R1), F22
	FMOVD   256(R1), F23
	FMOVD   264(R1), F24
	FMOVD   272(R1), F25
	FMOVD   280(R1), F26
	FMOVD   288(R1), F27
	FMOVD   296(R1), F28
	FMOVD   304(R1), F29
	FMOVD   312(R1), F30
	FMOVD   320(R1), F31

	ADD     $336, R1
	MOVD    8(R1), R10
	MOVFL   R10, $0xff // Restore of CR
	MOVD    16(R1), R10	// needed?
	MOVD    R10, LR
	RET

noswitch:
	BL      runtime·racecallback(SB)
	JMP     ret

// tls_g, g value for each thread in TLS
GLOBL runtime·tls_g+0(SB), TLSBSS+DUPOK, $8