Fix single-threaded recursive mutex locking (#777)

I have found this tiny mistake while investigating why
pthread_mutex_lock return EDEADLK for recursive mutexes in
single-threaded runtimes.

Author: Destructor17

libc-top-half/musl/src/thread/single-threaded/pthread_mutex_lock.c

@@ -8,7 +8,7 @@ int __pthread_mutex_lock(pthread_mutex_t *m)
 		1 - recursive
 		2 - errorcheck
 	*/
-	if (m->_m_type&3 != PTHREAD_MUTEX_RECURSIVE) {
+	if ((m->_m_type & 3) != PTHREAD_MUTEX_RECURSIVE) {
 		if (m->_m_count) return EDEADLK;
 		m->_m_count = 1;
 	} else {

libc-top-half/musl/src/thread/single-threaded/pthread_mutex_trylock.c

@@ -8,7 +8,7 @@ int __pthread_mutex_trylock(pthread_mutex_t *m)
 		1 - recursive
 		2 - errorcheck
 	*/
-	if (m->_m_type&3 != PTHREAD_MUTEX_RECURSIVE) {
+	if ((m->_m_type & 3) != PTHREAD_MUTEX_RECURSIVE) {
 		if (m->_m_count) return EBUSY;
 		m->_m_count = 1;
 	} else {

test/CMakeLists.txt

@@ -341,6 +341,9 @@ if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 20.0)
   set_tests_properties(setjmp.wasm PROPERTIES LABELS v8fail)
 endif()
 
+# This test works for single-threaded targets too
+add_wasilibc_test(pthread_recursive_mutex.c)
+
 if (TARGET_TRIPLE MATCHES "-threads")
   add_wasilibc_test(busywait.c)
   add_wasilibc_test(pthread_cond_busywait.c)

test/src/pthread_recursive_mutex.c

@@ -0,0 +1,45 @@
+#include "test.h"
+#include <pthread.h>
+#include <stdlib.h>
+
+#define TEST(c, i)                                                             \
+  do {                                                                         \
+    int ret = c;                                                               \
+    if (ret != 0)                                                              \
+      t_error("%s failed (i = %d, returned %d)\n", #c, i, ret);                \
+  } while (0)
+
+int ITERS = 4;
+
+int main(void) {
+  pthread_mutex_t recursive_mutex;
+  pthread_mutexattr_t attr;
+
+  // Create a recursive mutex
+  pthread_mutexattr_init(&attr);
+  pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+  pthread_mutex_init(&recursive_mutex, &attr);
+  pthread_mutexattr_destroy(&attr);
+
+  // Lock the mutex ITERS times
+  // Deadlock can't occur since all locks are held on the same thread
+  for (int i = 0; i < ITERS; i++) {
+    TEST(pthread_mutex_lock(&recursive_mutex), i);
+  }
+  // Unlock the mutex ITERS times
+  for (int i = 0; i < ITERS; i++) {
+    TEST(pthread_mutex_unlock(&recursive_mutex), i);
+  }
+
+  // Lock the mutex ITERS times without blocking
+  for (int i = 0; i < ITERS; i++) {
+    TEST(pthread_mutex_trylock(&recursive_mutex), i);
+  }
+  // Unlock the mutex ITERS times again
+  for (int i = 0; i < ITERS; i++) {
+    TEST(pthread_mutex_unlock(&recursive_mutex), i);
+  }
+
+  pthread_mutex_destroy(&recursive_mutex);
+  return t_status;
+}