threads: implement init of TLS and stack pointer (#342)

* threads: implement init of TLS and stack pointer

* fix: rename wasi_snapshot_preview2_thread_spawn to wasi_thread_spawn

Signed-off-by: Harald Hoyer <harald@profian.com>

* fix: change signature of wasi_thread_start

Signed-off-by: Harald Hoyer <harald@profian.com>

* fix: pthread_exit for WASI

Can't use `exit()` because it is too high level.
Have to unlock the thread list.

Signed-off-by: Harald Hoyer <harald@profian.com>

* fix: initialize struct pthread for the main thread

Signed-off-by: Harald Hoyer <harald@profian.com>

* fix: store the aligned stack minus `struct start_args`

Signed-off-by: Harald Hoyer <harald@profian.com>

Signed-off-by: Harald Hoyer <harald@profian.com>
Co-authored-by: Harald Hoyer <harald@profian.com>

Author: 12101111

Makefile

@@ -192,9 +192,16 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
 ifeq ($(THREAD_MODEL), posix)
 LIBC_TOP_HALF_MUSL_SOURCES += \
     $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
+        env/__init_tls.c \
+        stdio/__lockfile.c \
         thread/__lock.c \
         thread/__wait.c \
         thread/__timedwait.c \
+        thread/default_attr.c \
+        thread/pthread_attr_destroy.c \
+        thread/pthread_attr_init.c \
+        thread/pthread_attr_setstack.c \
+        thread/pthread_attr_setstacksize.c \
         thread/pthread_cleanup_push.c \
         thread/pthread_cond_broadcast.c \
         thread/pthread_cond_destroy.c \
@@ -235,6 +242,7 @@ LIBC_TOP_HALF_MUSL_SOURCES += \
         thread/pthread_rwlockattr_init.c \
         thread/pthread_rwlockattr_setpshared.c \
         thread/pthread_setcancelstate.c \
+        thread/pthread_self.c \
         thread/pthread_testcancel.c \
         thread/sem_destroy.c \
         thread/sem_getvalue.c \

expected/wasm32-wasi/posix/defined-symbols.txt

@@ -23,6 +23,7 @@ __c_locale
 __clock
 __clock_gettime
 __clock_nanosleep
+__copy_tls
 __cos
 __cosdf
 __cosl
@@ -38,6 +39,8 @@ __ctype_tolower_loc
 __ctype_toupper_loc
 __cxa_atexit
 __cxa_finalize
+__default_guardsize
+__default_stacksize
 __des_setkey
 __do_cleanup_pop
 __do_cleanup_push
@@ -87,6 +90,7 @@ __getopt_msg
 __gmtime_r
 __hwcap
 __inet_aton
+__init_tp
 __intscan
 __invtrigl_R
 __isalnum_l
@@ -144,6 +148,7 @@ __locale_lock
 __locale_lockptr
 __localtime_r
 __lock
+__lockfile
 __log2_data
 __log2f_data
 __log_data
@@ -265,6 +270,7 @@ __tan
 __tandf
 __tanl
 __testcancel
+__thread_list_lock
 __timedwait
 __timedwait_cp
 __tl_lock
@@ -288,6 +294,7 @@ __tsearch_balance
 __uflow
 __unlist_locked_file
 __unlock
+__unlockfile
 __uselocale
 __utc
 __wait
@@ -318,6 +325,7 @@ __wasi_fd_seek
 __wasi_fd_sync
 __wasi_fd_tell
 __wasi_fd_write
+__wasi_init_tp
 __wasi_path_create_directory
 __wasi_path_filestat_get
 __wasi_path_filestat_set_times
@@ -371,6 +379,7 @@ __wasilibc_nocwd_scandirat
 __wasilibc_nocwd_symlinkat
 __wasilibc_nocwd_utimensat
 __wasilibc_open_nomode
+__wasilibc_pthread_self
 __wasilibc_register_preopened_fd
 __wasilibc_rename_newat
 __wasilibc_rename_oldat
@@ -953,6 +962,10 @@ program_invocation_name
 program_invocation_short_name
 pselect
 psignal
+pthread_attr_destroy
+pthread_attr_init
+pthread_attr_setstack
+pthread_attr_setstacksize
 pthread_cond_broadcast
 pthread_cond_destroy
 pthread_cond_init
@@ -992,6 +1005,7 @@ pthread_rwlock_wrlock
 pthread_rwlockattr_destroy
 pthread_rwlockattr_init
 pthread_rwlockattr_setpshared
+pthread_self
 pthread_setcancelstate
 pthread_testcancel
 pthread_timedjoin_np
@@ -1182,6 +1196,7 @@ tfind
 tgamma
 tgammaf
 tgammal
+thrd_current
 thrd_sleep
 time
 timegm

expected/wasm32-wasi/posix/undefined-symbols.txt

@@ -1,7 +1,4 @@
 __addtf3
-__copy_tls
-__default_guardsize
-__default_stacksize
 __divtf3
 __eqtf2
 __extenddftf2
@@ -59,19 +56,18 @@ __imported_wasi_snapshot_preview1_sock_accept
 __imported_wasi_snapshot_preview1_sock_recv
 __imported_wasi_snapshot_preview1_sock_send
 __imported_wasi_snapshot_preview1_sock_shutdown
-__imported_wasi_snapshot_preview2_thread_spawn
+__imported_wasi_thread_spawn
 __letf2
-__lockfile
 __lttf2
 __main_argc_argv
 __netf2
 __stack_pointer
 __subtf3
-__thread_list_lock
+__tls_align
 __tls_base
+__tls_size
 __trunctfdf2
 __trunctfsf2
-__unlockfile
 __unordtf2
-__wasilibc_pthread_self
 __wasm_call_ctors
+__wasm_init_tls

libc-bottom-half/crt/crt1-command.c

@@ -1,5 +1,6 @@
 #ifdef _REENTRANT
 #include <stdatomic.h>
+extern void __wasi_init_tp(void);
 #endif
 #include <wasi/api.h>
 extern void __wasm_call_ctors(void);
@@ -29,6 +30,10 @@ void _start(void) {
     started = 1;
 #endif
 
+#ifdef _REENTRANT
+	__wasi_init_tp();
+#endif
+
     // The linker synthesizes this to call constructors.
     __wasm_call_ctors();
 

libc-bottom-half/sources/__wasilibc_real.c

@@ -660,13 +660,13 @@ __wasi_errno_t __wasi_sock_shutdown(
 }
 
 #ifdef _REENTRANT
-int32_t __imported_wasi_snapshot_preview2_thread_spawn(int32_t arg0) __attribute__((
-    __import_module__("wasi_snapshot_preview2"),
+int32_t __imported_wasi_thread_spawn(int32_t arg0) __attribute__((
+    __import_module__("wasi"),
     __import_name__("thread_spawn")
 ));
 
 __wasi_errno_t __wasi_thread_spawn(void* start_arg) {
-	int32_t ret = __imported_wasi_snapshot_preview2_thread_spawn((int32_t) start_arg);
+	int32_t ret = __imported_wasi_thread_spawn((int32_t) start_arg);
     return (uint16_t) ret;
 }
 #endif

libc-top-half/musl/arch/wasm32/pthread_arch.h

@@ -1,11 +1,5 @@
-static inline uintptr_t __get_tp(void) {
-#if _REENTRANT
-  int val;
-  __asm__("global.get __wasilibc_pthread_self\n"
-          "local.set %0"
-          : "=r"(val));
-  return val;
-#else
-  return 0;
-#endif
+extern _Thread_local struct __pthread __wasilibc_pthread_self;
+
+static inline uintptr_t __get_tp() {
+  return (uintptr_t)&__wasilibc_pthread_self;
 }

libc-top-half/musl/src/env/__init_tls.c

@@ -1,7 +1,11 @@
+#ifdef __wasilibc_unmodified_upstream
 #define SYSCALL_NO_TLS 1
 #include <elf.h>
+#endif
 #include <limits.h>
+#ifdef __wasilibc_unmodified_upstream
 #include <sys/mman.h>
+#endif
 #include <string.h>
 #include <stddef.h>
 #include "pthread_impl.h"
@@ -11,22 +15,32 @@
 
 volatile int __thread_list_lock;
 
+#ifndef __wasilibc_unmodified_upstream
+void __wasi_init_tp() {
+	__init_tp((void *)__get_tp());
+}
+#endif
+
 int __init_tp(void *p)
 {
 	pthread_t td = p;
 	td->self = td;
+#ifdef __wasilibc_unmodified_upstream
 	int r = __set_thread_area(TP_ADJ(p));
 	if (r < 0) return -1;
 	if (!r) libc.can_do_threads = 1;
 	td->detach_state = DT_JOINABLE;
 	td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock);
+#endif
 	td->locale = &libc.global_locale;
 	td->robust_list.head = &td->robust_list.head;
 	td->sysinfo = __sysinfo;
 	td->next = td->prev = td;
 	return 0;
 }
 
+#ifdef __wasilibc_unmodified_upstream
+
 static struct builtin_tls {
 	char c;
 	struct pthread pt;
@@ -35,9 +49,15 @@ static struct builtin_tls {
 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
 
 static struct tls_module main_tls;
+#endif
+
+#ifndef __wasilibc_unmodified_upstream
+extern void __wasm_init_tls(void*);
+#endif
 
 void *__copy_tls(unsigned char *mem)
 {
+#ifdef __wasilibc_unmodified_upstream
 	pthread_t td;
 	struct tls_module *p;
 	size_t i;
@@ -69,8 +89,20 @@ void *__copy_tls(unsigned char *mem)
 	dtv[0] = libc.tls_cnt;
 	td->dtv = dtv;
 	return td;
+#else
+	size_t tls_align = __builtin_wasm_tls_align();
+	volatile void* tls_base = __builtin_wasm_tls_base();
+	mem += tls_align;
+	mem -= (uintptr_t)mem & (tls_align-1);
+	__wasm_init_tls(mem);
+  	__asm__("local.get %0\n"
+			"global.set __tls_base\n"
+			:: "r"(tls_base));
+	return mem;
+#endif
 }
 
+#ifdef __wasilibc_unmodified_upstream
 #if ULONG_MAX == 0xffffffff
 typedef Elf32_Phdr Phdr;
 #else
@@ -151,3 +183,4 @@ static void static_init_tls(size_t *aux)
 }
 
 weak_alias(static_init_tls, __init_tls);
+#endif

libc-top-half/musl/src/internal/libc.h

@@ -18,8 +18,10 @@ struct tls_module {
 };
 
 struct __libc {
-#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
+#ifdef __wasilibc_unmodified_upstream
 	char can_do_threads;
+#endif
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
 	char threaded;
 #endif
 #ifdef __wasilibc_unmodified_upstream // WASI doesn't currently use any code that needs "secure" mode
@@ -32,7 +34,7 @@ struct __libc {
 #ifdef __wasilibc_unmodified_upstream // WASI has no auxv
 	size_t *auxv;
 #endif
-#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
+#ifdef __wasilibc_unmodified_upstream // WASI use different TLS implement
 	struct tls_module *tls_head;
 	size_t tls_size, tls_align, tls_cnt;
 #endif

libc-top-half/musl/src/internal/pthread_impl.h

@@ -25,8 +25,10 @@ struct pthread {
 	/* Part 1 -- these fields may be external or
 	 * internal (accessed via asm) ABI. Do not change. */
 	struct pthread *self;
+#ifdef __wasilibc_unmodified_upstream
 #ifndef TLS_ABOVE_TP
 	uintptr_t *dtv;
+#endif
 #endif
 	struct pthread *prev, *next; /* non-ABI */
 	uintptr_t sysinfo;