Wasip3: File handing implementation (WebAssembly#757)

This is an initial implementation of some basic file-handling functions
such as reading, writing, opening directories, iterating them, etc. A
number of previously failing tests are now passing and the
`wasip2_file.c` implementation is now merged with WASIp3 with `file.c`.
Bindings generation is also updated to flag some specific functions as
being lowered synchronously as libc always blocks on the result.

Note that much of this PR is primarily authored by @cpetig. I've
rebased, squashed, and applied my own review comments from WebAssembly#720 to this.
Notably some small edge cases here and there are handled and like with
other files `wasip{2,3}_file.c` has now simply become `file.c` with both
versions living in the same file.

Co-authored-by: Christof Petig <christof.petig@arcor.de>

Author: alexcrichton

cmake/bindings.cmake

@@ -107,6 +107,17 @@ add_custom_target(
       --rename wasi:cli/terminal-stdin@${wasip3-version}=terminal_stdin
       --rename wasi:cli/terminal-stdout@${wasip3-version}=terminal_stdout
       --rename wasi:cli/terminal-stderr@${wasip3-version}=terminal_stderr
+
+      # Disable async bindings generation for some functions which are only
+      # ever called synchronously within libc.
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.metadata-hash-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.stat"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.get-flags"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.open-at"
+      "--async=-wasi:filesystem/types@${wasip3-version}#[method]descriptor.read-directory"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-until"
+      "--async=-wasi:clocks/monotonic-clock@${wasip3-version}#wait-for"
       ${CMAKE_SOURCE_DIR}/wasi/p3/wit
   COMMAND cmake -E copy wasip3.h ${bottom_half}/headers/public/wasi/__generated_wasip3.h
   COMMAND cmake -E copy wasip3_component_type.o ${bottom_half}/sources

expected/wasm32-wasip2/defined-symbols.txt

@@ -1494,6 +1494,7 @@ vwprintf
 vwscanf
 wall_clock_now
 wall_clock_resolution
+wasi_string_from_c
 wasip2_list_string_free
 wasip2_list_tuple2_string_string_free
 wasip2_list_u32_free
@@ -1502,7 +1503,6 @@ wasip2_option_string_free
 wasip2_string_dup
 wasip2_string_dup_n
 wasip2_string_free
-wasip2_string_from_c
 wasip2_string_set
 wasip2_tuple2_string_string_free
 wcpcpy

expected/wasm32-wasip3/defined-symbols.txt

@@ -302,6 +302,7 @@ __wasilibc_fd_renumber
 __wasilibc_find_abspath
 __wasilibc_find_relpath
 __wasilibc_find_relpath_alloc
+__wasilibc_future_block_on
 __wasilibc_get_environ
 __wasilibc_get_service_entry_by_name
 __wasilibc_get_service_entry_by_port
@@ -339,6 +340,8 @@ __wasilibc_rmdirat
 __wasilibc_sockaddr_to_wasi
 __wasilibc_sockaddr_validate
 __wasilibc_stat
+__wasilibc_stream_block_on
+__wasilibc_subtask_block_on_and_drop
 __wasilibc_tell
 __wasilibc_unlinkat
 __wasilibc_unspecified_addr
@@ -1463,6 +1466,7 @@ vswprintf
 vswscanf
 vwprintf
 vwscanf
+wasi_string_from_c
 wasip3_backpressure_dec
 wasip3_backpressure_inc
 wasip3_context_get_0
@@ -1477,7 +1481,6 @@ wasip3_string_dup
 wasip3_string_dup_n
 wasip3_string_free
 wasip3_string_set
-wasip3_subtask_block_on
 wasip3_subtask_cancel
 wasip3_subtask_drop
 wasip3_task_cancel
@@ -1495,7 +1498,6 @@ wasip3_thread_yield_cancellable
 wasip3_thread_yield_to_suspended
 wasip3_thread_yield_to_suspended_cancellable
 wasip3_tuple2_string_string_free
-wasip3_waitable_block_on
 wasip3_waitable_join
 wasip3_waitable_set_drop
 wasip3_waitable_set_new

libc-bottom-half/CMakeLists.txt

@@ -140,6 +140,7 @@ else()
     sources/getsockpeername.c
     sources/listen.c
     sources/file_utils.c
+    sources/file.c
     sources/recv.c
     sources/send.c
     sources/shutdown.c
@@ -155,7 +156,6 @@ if(WASI STREQUAL "p2")
   list(APPEND bottom_half_sources
     sources/netdb.c
     sources/wasip2.c
-    sources/wasip2_file.c
     sources/wasip2_stdio.c
   )
 endif()
@@ -164,7 +164,6 @@ if (WASI STREQUAL "p3")
   list(APPEND bottom_half_sources
     sources/wasip3.c
     sources/wasip3_block_on.c
-    sources/wasip3_file.c
     sources/wasip3_stdio.c
   )
 endif()

libc-bottom-half/cloudlibc/src/libc/dirent/dirent_impl.h

@@ -28,7 +28,7 @@ struct _DIR {
   size_t skip;
   size_t offset;
 #elif defined(__wasip3__)
-  filesystem_stream_directory_entry_t stream;
+  filesystem_tuple2_stream_directory_entry_future_result_void_error_code_t stream;
   size_t skip;
   size_t offset;
 #else
@@ -40,4 +40,27 @@ struct _DIR {
   size_t dirent_size;
 };
 
+static inline void dirent_close_streams(DIR *dirp) {
+#if defined(__wasip1__)
+  (void) dirp;
+  // nothing to close ...
+#elif defined(__wasip2__)
+  if (dirp->stream.__handle) {
+    filesystem_directory_entry_stream_drop_own(dirp->stream);
+    dirp->stream.__handle = 0;
+  }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 != 0) {
+    filesystem_stream_directory_entry_drop_readable(dirp->stream.f0);
+    dirp->stream.f0 = 0;
+  }
+  if (dirp->stream.f1 != 0) {
+    filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+    dirp->stream.f1 = 0;
+  }
+#else
+# error "Unknown WASI version"
+#endif
+}
+
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/fdclosedir.c

@@ -9,17 +9,10 @@
 
 int fdclosedir(DIR *dirp) {
   int fd = dirp->fd;
-#if defined(__wasip1__)
+#ifdef __wasip1__
   free(dirp->buffer);
-#elif defined(__wasip2__)
-  if (dirp->stream.__handle != 0)
-    filesystem_directory_entry_stream_drop_own(dirp->stream);
-#elif defined(__wasip3__)
-  if (dirp->stream != 0)
-    filesystem_stream_directory_entry_drop_readable(dirp->stream);
-#else
-# error "Unsupported WASI version"
 #endif
+  dirent_close_streams(dirp);
   free(dirp->dirent);
   free(dirp);
   return fd;

libc-bottom-half/cloudlibc/src/libc/dirent/fdopendir.c

@@ -49,7 +49,7 @@ DIR *fdopendir(int fd) {
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
   // Translate the file descriptor to an internal handle
   filesystem_borrow_descriptor_t file_handle;
   if (fd_to_file_handle(fd, &file_handle) < 0) {
@@ -58,6 +58,7 @@ DIR *fdopendir(int fd) {
   }
 
   // Read the directory
+#if defined(__wasip2__)
   filesystem_own_directory_entry_stream_t result;
   filesystem_error_code_t error_code;
   bool ok = filesystem_method_descriptor_read_directory(file_handle,
@@ -69,19 +70,16 @@ DIR *fdopendir(int fd) {
     return NULL;
   }
 
-  dirp->fd = fd;
   dirp->stream = result;
+#elif defined(__wasip3__)
+  filesystem_method_descriptor_read_directory(file_handle, &dirp->stream);
+#endif
+  dirp->fd = fd;
   dirp->skip = 0;
   dirp->offset = 0;
   dirp->dirent = NULL;
   dirp->dirent_size = 1;
   return dirp;
-#elif defined(__wasip3__)
-  (void) fd;
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
 #else
 # error "Unsupported WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c

@@ -31,8 +31,11 @@ static_assert(DT_REG == __WASI_FILETYPE_REGULAR_FILE, "Value mismatch");
 static_assert(DT_UNKNOWN == __WASI_FILETYPE_UNKNOWN, "Value mismatch");
 #endif
 
+#ifdef __wasip3__
+#include <wasi/wasip3_block.h>
+#endif
+
 // Grows a buffer to be large enough to hold a certain amount of data.
-#ifndef __wasip3__
 static struct dirent* grow(struct dirent **buffer, size_t *buffer_size, size_t target_size) {
   if (*buffer_size < target_size) {
     size_t new_size = *buffer_size;
@@ -48,7 +51,6 @@ static struct dirent* grow(struct dirent **buffer, size_t *buffer_size, size_t t
   }
   return *buffer;
 }
-#endif
 
 #if defined(__wasip1__)
 struct dirent *readdir(DIR *dirp) {
@@ -144,12 +146,13 @@ struct dirent *readdir(DIR *dirp) {
   }
 }
 
-#elif defined(__wasip2__)
+#elif defined(__wasip2__) || defined(__wasip3__)
 
 static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t *handle) {
   if (fd_to_file_handle(dirp->fd, handle) < 0)
     return -1;
 
+#ifdef __wasip2__
   if (dirp->stream.__handle != 0)
     return 0;
 
@@ -161,10 +164,15 @@ static int ensure_has_directory_stream(DIR *dirp, filesystem_borrow_descriptor_t
     translate_error(error_code);
     return -1;
   }
+#elif defined(__wasip3__)
+  if (dirp->stream.f0 == 0)
+    filesystem_method_descriptor_read_directory(*handle, &dirp->stream);
+#endif
   return 0;
 }
 
 static struct dirent *readdir_next(DIR *dirp) {
+  bool ok;
   filesystem_metadata_hash_value_t metadata;
   filesystem_error_code_t error_code;
   filesystem_borrow_descriptor_t dir_handle;
@@ -178,9 +186,9 @@ static struct dirent *readdir_next(DIR *dirp) {
     dirp->offset += 1;
     if (grow(&dirp->dirent, &dirp->dirent_size, offsetof(struct dirent, d_name) + 2) == NULL)
       return NULL;
-    bool ok = filesystem_method_descriptor_metadata_hash(dir_handle,
-                                                         &metadata,
-                                                         &error_code);
+    ok = filesystem_method_descriptor_metadata_hash(dir_handle,
+                                                    &metadata,
+                                                    &error_code);
     if (!ok) {
       translate_error(error_code);
       return NULL;
@@ -206,11 +214,12 @@ static struct dirent *readdir_next(DIR *dirp) {
     return dirp->dirent;
   }
 
+#if defined(__wasip2__)
   filesystem_borrow_directory_entry_stream_t stream = filesystem_borrow_directory_entry_stream(dirp->stream);
   filesystem_option_directory_entry_t dir_entry_optional;
-  bool ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
-                                                                          &dir_entry_optional,
-                                                                          &error_code);
+  ok = filesystem_method_directory_entry_stream_read_directory_entry(stream,
+                                                                     &dir_entry_optional,
+                                                                     &error_code);
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -222,10 +231,54 @@ static struct dirent *readdir_next(DIR *dirp) {
 
   filesystem_directory_entry_t dir_entry = dir_entry_optional.val;
 
+#elif defined(__wasip3__)
+  filesystem_directory_entry_t dir_entry;
+
+  // Loop until at least one stream entry is read, or until the stream is closed.
+  bool closed = false;
+  while (1) {
+    size_t amount =
+      __wasilibc_stream_block_on(
+          filesystem_stream_directory_entry_read(dirp->stream.f0, &dir_entry, 1),
+          dirp->stream.f0,
+          &closed);
+
+    // If something was read, then break out and process that below.
+    if (amount > 0)
+      break;
+
+    // If nothing was read and the stream isn't finished yet, try again.
+    if (!closed)
+      continue;
+
+    // If the stream's result future hasn't been read yet, do so here.
+    if (dirp->stream.f1) {
+      filesystem_result_void_error_code_t result;
+      __wasilibc_future_block_on(
+          filesystem_future_result_void_error_code_read(dirp->stream.f1, &result),
+          dirp->stream.f1);
+      filesystem_future_result_void_error_code_drop_readable(dirp->stream.f1);
+      dirp->stream.f1 = 0;
+      if (result.is_err)
+        translate_error(result.val.err);
+    }
+
+    // The stream is closed, so return NULL. This'll set `errno` based on the
+    // result of the future above.
+    return NULL;
+  }
+#else
+#error "Unknown WASI version"
+#endif
+
   // Ensure that the dirent is large enough to fit the filename
   size_t the_size = offsetof(struct dirent, d_name);
   if (grow(&dirp->dirent, &dirp->dirent_size, the_size + dir_entry.name.len + 1) == NULL) {
+#ifdef __wasip2__
     wasip2_string_free(&dir_entry.name);
+#else
+    wasip3_string_free(&dir_entry.name);
+#endif
     return NULL;
   }
 
@@ -241,7 +294,11 @@ static struct dirent *readdir_next(DIR *dirp) {
                                                      &dir_entry.name,
                                                      &metadata,
                                                      &error_code);
+#ifdef __wasip2__
   wasip2_string_free(&dir_entry.name);
+#else
+  wasip3_string_free(&dir_entry.name);
+#endif
   if (!ok) {
     translate_error(error_code);
     return NULL;
@@ -260,13 +317,6 @@ struct dirent *readdir(DIR *dirp) {
   }
   return result;
 }
-#elif defined(__wasip3__)
-struct dirent *readdir(DIR *dirp) {
-  // TODO(wasip3)
-  errno = ENOTSUP;
-  free(dirp);
-  return NULL;
-}
 #else
 # error "Unknown WASI version"
 #endif

libc-bottom-half/cloudlibc/src/libc/dirent/rewinddir.c

@@ -13,12 +13,8 @@ void rewinddir(DIR *dirp) {
   dirp->cookie = __WASI_DIRCOOKIE_START;
   // Mark entire buffer as processed to force a read of new data.
   dirp->buffer_used = dirp->buffer_processed = dirp->buffer_size;
-#elif defined(__wasip2__)
-  dirp->stream.__handle = 0;
-  dirp->skip = 0;
-  dirp->offset = 0;
-#elif defined(__wasip3__)
-  dirp->stream = 0;
+#elif defined(__wasip2__) || defined(__wasip3__)
+  dirent_close_streams(dirp);
   dirp->skip = 0;
   dirp->offset = 0;
 #else

libc-bottom-half/cloudlibc/src/libc/dirent/seekdir.c

@@ -13,12 +13,8 @@ void seekdir(DIR *dirp, long loc) {
   // Mark entire buffer as processed to force a read of new data.
   // TODO(ed): We could prevent a read if the offset is in the buffer.
   dirp->buffer_used = dirp->buffer_processed = dirp->buffer_size;
-#elif defined(__wasip2__)
-  dirp->stream.__handle = 0;
-  dirp->skip = loc;
-  dirp->offset = 0;
-#elif defined(__wasip3__)
-  dirp->stream = 0;
+#elif defined(__wasip2__) || defined(__wasip3__)
+  dirent_close_streams(dirp);
   dirp->skip = loc;
   dirp->offset = 0;
 #else