threads: implement support for pthread mutexes (#315)

This change adds pthread's mutex support to the `THREAD_MODEL=posix`
build of wasi-libc. Some less-common features are unsupported and
documented here:
- mutex robust lists are disabled and their use will return a runtime
  error; currently WASI does not support the concept of multiple
  processes so maintaining robust mutexes across processes does not yet
  make sense in wasi-libc
- timed locks with priority inheritance (PI) are disabled and will act
  as any other mutex; this feature is related to task priorities which
  is not yet relevant in the WASI ecosystem (see [priority-inheritance
  futexes](https://man7.org/linux/man-pages/man2/futex.2.html))
- thread cancellation is ignored; this feature is difficult to support
  and @sunfishcode would likely want to discuss this before adding it at
  some later time.

Author: abrown

Makefile

@@ -187,6 +187,29 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
                  $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/complex/*.c)) \
     $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/crypt/*.c)
 
+ifeq ($(THREAD_MODEL), posix)
+LIBC_TOP_HALF_MUSL_SOURCES += \
+    $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
+        thread/__wait.c \
+        thread/__timedwait.c \
+        thread/pthread_mutex_consistent.c \
+        thread/pthread_mutex_destroy.c \
+        thread/pthread_mutex_init.c \
+        thread/pthread_mutex_getprioceiling.c \
+        thread/pthread_mutex_lock.c \
+        thread/pthread_mutex_timedlock.c \
+        thread/pthread_mutex_trylock.c \
+        thread/pthread_mutex_unlock.c \
+        thread/pthread_mutexattr_destroy.c \
+        thread/pthread_mutexattr_init.c \
+        thread/pthread_mutexattr_setprotocol.c \
+        thread/pthread_mutexattr_setpshared.c \
+        thread/pthread_mutexattr_setrobust.c \
+        thread/pthread_mutexattr_settype.c \
+        thread/pthread_setcancelstate.c \
+    )
+endif
+
 MUSL_PRINTSCAN_SOURCES = \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal/floatscan.c \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/stdio/vfprintf.c \

expected/wasm32-wasi/posix/defined-symbols.txt

@@ -179,6 +179,12 @@ __pow_log_data
 __powf_log2_data
 __progname
 __progname_full
+__pthread_mutex_lock
+__pthread_mutex_timedlock
+__pthread_mutex_trylock
+__pthread_mutex_trylock_owner
+__pthread_mutex_unlock
+__pthread_setcancelstate
 __putenv
 __qsort_r
 __rand48_step
@@ -237,6 +243,8 @@ __sysv_signal
 __tan
 __tandf
 __tanl
+__timedwait
+__timedwait_cp
 __tm_to_secs
 __tm_to_tzname
 __tolower_l
@@ -256,6 +264,7 @@ __uflow
 __unlist_locked_file
 __uselocale
 __utc
+__wait
 __wasi_args_get
 __wasi_args_sizes_get
 __wasi_clock_res_get
@@ -313,6 +322,7 @@ __wasilibc_fd_renumber
 __wasilibc_find_abspath
 __wasilibc_find_relpath
 __wasilibc_find_relpath_alloc
+__wasilibc_futex_wait
 __wasilibc_get_environ
 __wasilibc_iftodt
 __wasilibc_initialize_environ
@@ -914,6 +924,21 @@ program_invocation_name
 program_invocation_short_name
 pselect
 psignal
+pthread_mutex_consistent
+pthread_mutex_destroy
+pthread_mutex_getprioceiling
+pthread_mutex_init
+pthread_mutex_lock
+pthread_mutex_timedlock
+pthread_mutex_trylock
+pthread_mutex_unlock
+pthread_mutexattr_destroy
+pthread_mutexattr_init
+pthread_mutexattr_setprotocol
+pthread_mutexattr_setpshared
+pthread_mutexattr_setrobust
+pthread_mutexattr_settype
+pthread_setcancelstate
 putc
 putc_unlocked
 putchar

expected/wasm32-wasi/posix/undefined-symbols.txt

@@ -72,4 +72,3 @@ __unlockfile
 __unordtf2
 __wasilibc_pthread_self
 __wasm_call_ctors
-pthread_setcancelstate

libc-top-half/musl/src/internal/pthread_impl.h

@@ -170,6 +170,9 @@ hidden int __libc_sigaction(int, const struct sigaction *, struct sigaction *);
 #endif
 hidden void __unmapself(void *, size_t);
 
+#ifndef __wasilibc_unmodified_upstream
+hidden int __wasilibc_futex_wait(volatile void *, int, int, int64_t);
+#endif
 hidden int __timedwait(volatile int *, int, clockid_t, const struct timespec *, int);
 hidden int __timedwait_cp(volatile int *, int, clockid_t, const struct timespec *, int);
 hidden void __wait(volatile int *, volatile int *, int, int);

libc-top-half/musl/src/internal/syscall.h

@@ -1,3 +1,4 @@
+#ifdef __wasilibc_unmodified_upstream
 #ifndef _INTERNAL_SYSCALL_H
 #define _INTERNAL_SYSCALL_H
 
@@ -396,3 +397,4 @@ hidden void __procfdname(char __buf[static 15+3*sizeof(int)], unsigned);
 hidden void *__vdsosym(const char *, const char *);
 
 #endif
+#endif

libc-top-half/musl/src/thread/__timedwait.c

@@ -5,6 +5,7 @@
 #include "syscall.h"
 #include "pthread_impl.h"
 
+#ifdef __wasilibc_unmodified_upstream
 #define IS32BIT(x) !((x)+0x80000000ULL>>32)
 #define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63))
 
@@ -28,6 +29,16 @@ static int __futex4_cp(volatile void *addr, int op, int val, const struct timesp
 
 static volatile int dummy = 0;
 weak_alias(dummy, __eintr_valid_flag);
+#else
+static int __futex4_cp(volatile void *addr, int op, int val, const struct timespec *to)
+{
+	int64_t max_wait_ns = -1;
+	if (to) {
+		max_wait_ns = (int64_t)(to->tv_sec * 1000000000 + to->tv_nsec);
+	}
+	return __wasilibc_futex_wait(addr, op, val, max_wait_ns);
+}
+#endif
 
 int __timedwait_cp(volatile int *addr, int val,
 	clockid_t clk, const struct timespec *at, int priv)
@@ -51,11 +62,13 @@ int __timedwait_cp(volatile int *addr, int val,
 
 	r = -__futex4_cp(addr, FUTEX_WAIT|priv, val, top);
 	if (r != EINTR && r != ETIMEDOUT && r != ECANCELED) r = 0;
+#ifdef __wasilibc_unmodified_upstream
 	/* Mitigate bug in old kernels wrongly reporting EINTR for non-
 	 * interrupting (SA_RESTART) signal handlers. This is only practical
 	 * when NO interrupting signal handlers have been installed, and
 	 * works by sigaction tracking whether that's the case. */
 	if (r == EINTR && !__eintr_valid_flag) r = 0;
+#endif
 
 	return r;
 }

libc-top-half/musl/src/thread/__wait.c

@@ -1,4 +1,38 @@
 #include "pthread_impl.h"
+#ifndef __wasilibc_unmodified_upstream
+#include "assert.h"
+#endif
+
+#ifndef __wasilibc_unmodified_upstream
+// Use WebAssembly's `wait` instruction to implement a futex. Note that `op` is
+// unused but retained as a parameter to match the original signature of the
+// syscall and that, for `max_wait_ns`, -1 (or any negative number) means wait
+// indefinitely.
+//
+// Adapted from Emscripten: see
+// https://github.com/emscripten-core/emscripten/blob/058a9fff/system/lib/pthread/emscripten_futex_wait.c#L111-L150.
+int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait_ns)
+{
+    if ((((intptr_t)addr) & 3) != 0) {
+        return -EINVAL;
+    }
+
+    int ret = __builtin_wasm_memory_atomic_wait32((int *)addr, val, max_wait_ns);
+
+    // memory.atomic.wait32 returns:
+    //   0 => "ok", woken by another agent.
+    //   1 => "not-equal", loaded value != expected value
+    //   2 => "timed-out", the timeout expired
+    if (ret == 1) {
+        return -EWOULDBLOCK;
+    }
+    if (ret == 2) {
+        return -ETIMEDOUT;
+    }
+    assert(ret == 0);
+    return 0;
+}
+#endif
 
 void __wait(volatile int *addr, volatile int *waiters, int val, int priv)
 {
@@ -10,8 +44,12 @@ void __wait(volatile int *addr, volatile int *waiters, int val, int priv)
 	}
 	if (waiters) a_inc(waiters);
 	while (*addr==val) {
+#ifdef __wasilibc_unmodified_upstream
 		__syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0) != -ENOSYS
 		|| __syscall(SYS_futex, addr, FUTEX_WAIT, val, 0);
+#else
+		__wasilibc_futex_wait(addr, FUTEX_WAIT, val, 0);
+#endif
 	}
 	if (waiters) a_dec(waiters);
 }

libc-top-half/musl/src/thread/pthread_mutex_destroy.c

@@ -2,9 +2,17 @@
 
 int pthread_mutex_destroy(pthread_mutex_t *mutex)
 {
+#ifdef __wasilibc_unmodified_upstream
 	/* If the mutex being destroyed is process-shared and has nontrivial
 	 * type (tracking ownership), it might be in the pending slot of a
 	 * robust_list; wait for quiescence. */
 	if (mutex->_m_type > 128) __vm_wait();
+#else
+	/* For now, wasi-libc chooses to avoid implementing robust mutex support
+	 * though this could be added later. The error code indicates that the
+	 * mutex was an invalid type, but it would be more accurate as 
+	 * "unimplemented". */
+	if (mutex->_m_type > 128) return EINVAL;
+#endif
 	return 0;
 }

libc-top-half/musl/src/thread/pthread_mutex_timedlock.c

@@ -1,5 +1,6 @@
 #include "pthread_impl.h"
 
+#ifdef __wasilibc_unmodified_upstream
 #define IS32BIT(x) !((x)+0x80000000ULL>>32)
 #define CLAMP(x) (int)(IS32BIT(x) ? (x) : 0x7fffffffU+((0ULL+(x))>>63))
 
@@ -52,6 +53,7 @@ static int pthread_mutex_timedlock_pi(pthread_mutex_t *restrict m, const struct
 	while (e != ETIMEDOUT);
 	return e;
 }
+#endif
 
 int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec *restrict at)
 {
@@ -65,8 +67,10 @@ int __pthread_mutex_timedlock(pthread_mutex_t *restrict m, const struct timespec
 	r = __pthread_mutex_trylock(m);
 	if (r != EBUSY) return r;
 
+#ifdef __wasilibc_unmodified_upstream
 	if (type&8) return pthread_mutex_timedlock_pi(m, at);
-	
+#endif
+
 	int spins = 100;
 	while (spins-- && m->_m_lock && !m->_m_waiters) a_spin();
 

libc-top-half/musl/src/thread/pthread_mutex_trylock.c

@@ -27,7 +27,9 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 	if (type & 128) {
 		if (!self->robust_list.off) {
 			self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next;
+#ifdef __wasilibc_unmodified_upstream
 			__syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long));
+#endif
 		}
 		if (m->_m_waiters) tid |= 0x80000000;
 		self->robust_list.pending = &m->_m_next;
@@ -43,7 +45,9 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 success:
 	if ((type&8) && m->_m_waiters) {
 		int priv = (type & 128) ^ 128;
+#ifdef __wasilibc_unmodified_upstream
 		__syscall(SYS_futex, &m->_m_lock, FUTEX_UNLOCK_PI|priv);
+#endif
 		self->robust_list.pending = 0;
 		return (type&4) ? ENOTRECOVERABLE : EBUSY;
 	}