1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
|
#ifdef __wasilibc_unmodified_upstream
#define SYSCALL_NO_TLS 1
#include <elf.h>
#endif
#include <limits.h>
#ifdef __wasilibc_unmodified_upstream
#include <sys/mman.h>
#endif
#include <string.h>
#include <stddef.h>
#include "pthread_impl.h"
#include "libc.h"
#include "atomic.h"
#include "syscall.h"
volatile int __thread_list_lock;
#ifndef __wasilibc_unmodified_upstream
/* These symbols are generated by wasm-ld. __stack_high/__stack_low
* symbols are only available in LLVM v16 and higher, therefore they're
* defined as weak symbols and if not available, __heap_base/__data_end
* is used instead.
*
* TODO: remove usage of __heap_base/__data_end for stack size calculation
* once we drop support for LLVM v15 and older.
*/
extern unsigned char __heap_base;
extern unsigned char __data_end;
extern unsigned char __global_base;
extern weak unsigned char __stack_high;
extern weak unsigned char __stack_low;
struct stack_bounds {
void *base;
size_t size;
};
static inline struct stack_bounds get_stack_bounds()
{
struct stack_bounds bounds;
if (&__stack_high) {
bounds.base = &__stack_high;
bounds.size = &__stack_high - &__stack_low;
} else {
unsigned char *sp;
__asm__(
".globaltype __stack_pointer, i32\n"
"global.get __stack_pointer\n"
"local.set %0\n"
: "=r"(sp));
if (sp > &__global_base) {
bounds.base = &__heap_base;
bounds.size = &__heap_base - &__data_end;
} else {
bounds.base = &__global_base;
bounds.size = (size_t)&__global_base;
}
}
return bounds;
}
void __wasi_init_tp() {
__init_tp((void *)__get_tp());
}
#endif
int __init_tp(void *p)
{
pthread_t td = p;
td->self = td;
#ifdef __wasilibc_unmodified_upstream
int r = __set_thread_area(TP_ADJ(p));
if (r < 0) return -1;
if (!r) libc.can_do_threads = 1;
td->detach_state = DT_JOINABLE;
td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock);
#else
struct stack_bounds bounds = get_stack_bounds();
__default_stacksize =
bounds.size < DEFAULT_STACK_MAX ?
bounds.size : DEFAULT_STACK_MAX;
td->detach_state = DT_JOINABLE;
td->stack = bounds.base;
td->stack_size = bounds.size;
td->guard_size = 0;
/*
* Initialize the TID to a value which doesn't conflict with
* host-allocated TIDs, so that TID-based locks can work.
*
* Note:
* - Host-allocated TIDs range from 1 to 0x1fffffff. (inclusive)
* - __tl_lock and __lockfile uses TID 0 as "unlocked".
* - __lockfile relies on the fact the most significant two bits
* of TIDs are 0.
*/
td->tid = 0x3fffffff;
#endif
td->locale = &libc.global_locale;
td->robust_list.head = &td->robust_list.head;
td->sysinfo = __sysinfo;
td->next = td->prev = td;
return 0;
}
#ifdef __wasilibc_unmodified_upstream
static struct builtin_tls {
char c;
struct pthread pt;
void *space[16];
} builtin_tls[1];
#define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
static struct tls_module main_tls;
#endif
#ifndef __wasilibc_unmodified_upstream
extern void __wasm_init_tls(void*);
#endif
void *__copy_tls(unsigned char *mem)
{
#ifdef __wasilibc_unmodified_upstream
pthread_t td;
struct tls_module *p;
size_t i;
uintptr_t *dtv;
#ifdef TLS_ABOVE_TP
dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1);
mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1);
td = (pthread_t)mem;
mem += sizeof(struct pthread);
for (i=1, p=libc.tls_head; p; i++, p=p->next) {
dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET;
memcpy(mem + p->offset, p->image, p->len);
}
#else
dtv = (uintptr_t *)mem;
mem += libc.tls_size - sizeof(struct pthread);
mem -= (uintptr_t)mem & (libc.tls_align-1);
td = (pthread_t)mem;
for (i=1, p=libc.tls_head; p; i++, p=p->next) {
dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
memcpy(mem - p->offset, p->image, p->len);
}
#endif
dtv[0] = libc.tls_cnt;
td->dtv = dtv;
return td;
#else
size_t tls_align = __builtin_wasm_tls_align();
volatile void* tls_base = __builtin_wasm_tls_base();
mem += tls_align;
mem -= (uintptr_t)mem & (tls_align-1);
__wasm_init_tls(mem);
__asm__("local.get %0\n"
"global.set __tls_base\n"
:: "r"(tls_base));
return mem;
#endif
}
#ifdef __wasilibc_unmodified_upstream
#if ULONG_MAX == 0xffffffff
typedef Elf32_Phdr Phdr;
#else
typedef Elf64_Phdr Phdr;
#endif
extern weak hidden const size_t _DYNAMIC[];
static void static_init_tls(size_t *aux)
{
unsigned char *p;
size_t n;
Phdr *phdr, *tls_phdr=0;
size_t base = 0;
void *mem;
for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) {
phdr = (void *)p;
if (phdr->p_type == PT_PHDR)
base = aux[AT_PHDR] - phdr->p_vaddr;
if (phdr->p_type == PT_DYNAMIC && _DYNAMIC)
base = (size_t)_DYNAMIC - phdr->p_vaddr;
if (phdr->p_type == PT_TLS)
tls_phdr = phdr;
if (phdr->p_type == PT_GNU_STACK &&
phdr->p_memsz > __default_stacksize)
__default_stacksize =
phdr->p_memsz < DEFAULT_STACK_MAX ?
phdr->p_memsz : DEFAULT_STACK_MAX;
}
if (tls_phdr) {
main_tls.image = (void *)(base + tls_phdr->p_vaddr);
main_tls.len = tls_phdr->p_filesz;
main_tls.size = tls_phdr->p_memsz;
main_tls.align = tls_phdr->p_align;
libc.tls_cnt = 1;
libc.tls_head = &main_tls;
}
main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image)
& (main_tls.align-1);
#ifdef TLS_ABOVE_TP
main_tls.offset = GAP_ABOVE_TP;
main_tls.offset += (-GAP_ABOVE_TP + (uintptr_t)main_tls.image)
& (main_tls.align-1);
#else
main_tls.offset = main_tls.size;
#endif
if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN;
libc.tls_align = main_tls.align;
libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread)
#ifdef TLS_ABOVE_TP
+ main_tls.offset
#endif
+ main_tls.size + main_tls.align
+ MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN;
if (libc.tls_size > sizeof builtin_tls) {
#ifndef SYS_mmap2
#define SYS_mmap2 SYS_mmap
#endif
mem = (void *)__syscall(
SYS_mmap2,
0, libc.tls_size, PROT_READ|PROT_WRITE,
MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
/* -4095...-1 cast to void * will crash on dereference anyway,
* so don't bloat the init code checking for error codes and
* explicitly calling a_crash(). */
} else {
mem = builtin_tls;
}
/* Failure to initialize thread pointer is always fatal. */
if (__init_tp(__copy_tls(mem)) < 0)
a_crash();
}
weak_alias(static_init_tls, __init_tls);
#endif
|
