summaryrefslogtreecommitdiffstats
path: root/src/runtime/memclr_arm64.s
blob: bef77651e48c445b4d88ab7eb31c330b872e54fd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "textflag.h"

// See memclrNoHeapPointers Go doc for important implementation constraints.

// func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16
	MOVD	ptr+0(FP), R0
	MOVD	n+8(FP), R1

	CMP	$16, R1
	// If n is equal to 16 bytes, use zero_exact_16 to zero
	BEQ	zero_exact_16

	// If n is greater than 16 bytes, use zero_by_16 to zero
	BHI	zero_by_16

	// n is less than 16 bytes
	ADD	R1, R0, R7
	TBZ	$3, R1, less_than_8
	MOVD	ZR, (R0)
	MOVD	ZR, -8(R7)
	RET

less_than_8:
	TBZ	$2, R1, less_than_4
	MOVW	ZR, (R0)
	MOVW	ZR, -4(R7)
	RET

less_than_4:
	CBZ	R1, ending
	MOVB	ZR, (R0)
	TBZ	$1, R1, ending
	MOVH	ZR, -2(R7)

ending:
	RET

zero_exact_16:
	// n is exactly 16 bytes
	STP	(ZR, ZR), (R0)
	RET

zero_by_16:
	// n greater than 16 bytes, check if the start address is aligned
	NEG	R0, R4
	ANDS	$15, R4, R4
	// Try zeroing using zva if the start address is aligned with 16
	BEQ	try_zva

	// Non-aligned store
	STP	(ZR, ZR), (R0)
	// Make the destination aligned
	SUB	R4, R1, R1
	ADD	R4, R0, R0
	B	try_zva

tail_maybe_long:
	CMP	$64, R1
	BHS	no_zva

tail63:
	ANDS	$48, R1, R3
	BEQ	last16
	CMPW	$32, R3
	BEQ	last48
	BLT	last32
	STP.P	(ZR, ZR), 16(R0)
last48:
	STP.P	(ZR, ZR), 16(R0)
last32:
	STP.P	(ZR, ZR), 16(R0)
	// The last store length is at most 16, so it is safe to use
	// stp to write last 16 bytes
last16:
	ANDS	$15, R1, R1
	CBZ	R1, last_end
	ADD	R1, R0, R0
	STP	(ZR, ZR), -16(R0)
last_end:
	RET

no_zva:
	SUB	$16, R0, R0
	SUB	$64, R1, R1

loop_64:
	STP	(ZR, ZR), 16(R0)
	STP	(ZR, ZR), 32(R0)
	STP	(ZR, ZR), 48(R0)
	STP.W	(ZR, ZR), 64(R0)
	SUBS	$64, R1, R1
	BGE	loop_64
	ANDS	$63, R1, ZR
	ADD	$16, R0, R0
	BNE	tail63
	RET

try_zva:
	// Try using the ZVA feature to zero entire cache lines
	// It is not meaningful to use ZVA if the block size is less than 64,
	// so make sure that n is greater than or equal to 64
	CMP	$63, R1
	BLE	tail63

	CMP	$128, R1
	// Ensure n is at least 128 bytes, so that there is enough to copy after
	// alignment.
	BLT	no_zva
	// Check if ZVA is allowed from user code, and if so get the block size
	MOVW	block_size<>(SB), R5
	TBNZ	$31, R5, no_zva
	CBNZ	R5, zero_by_line
	// DCZID_EL0 bit assignments
	// [63:5] Reserved
	// [4]    DZP, if bit set DC ZVA instruction is prohibited, else permitted
	// [3:0]  log2 of the block size in words, eg. if it returns 0x4 then block size is 16 words
	MRS	DCZID_EL0, R3
	TBZ	$4, R3, init
	// ZVA not available
	MOVW	$~0, R5
	MOVW	R5, block_size<>(SB)
	B	no_zva

init:
	MOVW	$4, R9
	ANDW	$15, R3, R5
	LSLW	R5, R9, R5
	MOVW	R5, block_size<>(SB)

	ANDS	$63, R5, R9
	// Block size is less than 64.
	BNE	no_zva

zero_by_line:
	CMP	R5, R1
	// Not enough memory to reach alignment
	BLO	no_zva
	SUB	$1, R5, R6
	NEG	R0, R4
	ANDS	R6, R4, R4
	// Already aligned
	BEQ	aligned

	// check there is enough to copy after alignment
	SUB	R4, R1, R3

	// Check that the remaining length to ZVA after alignment
	// is greater than 64.
	CMP	$64, R3
	CCMP	GE, R3, R5, $10  // condition code GE, NZCV=0b1010
	BLT	no_zva

	// We now have at least 64 bytes to zero, update n
	MOVD	R3, R1

loop_zva_prolog:
	STP	(ZR, ZR), (R0)
	STP	(ZR, ZR), 16(R0)
	STP	(ZR, ZR), 32(R0)
	SUBS	$64, R4, R4
	STP	(ZR, ZR), 48(R0)
	ADD	$64, R0, R0
	BGE	loop_zva_prolog

	ADD	R4, R0, R0

aligned:
	SUB	R5, R1, R1

loop_zva:
	WORD	$0xd50b7420 // DC ZVA, R0
	ADD	R5, R0, R0
	SUBS	R5, R1, R1
	BHS	loop_zva
	ANDS	R6, R1, R1
	BNE	tail_maybe_long
	RET

GLOBL block_size<>(SB), NOPTR, $8