Build and test Python with WASIp3 in CI (#784)

This is intended to mirror what's done for WASIp{1,2} where Python is
built against an in-tree copy of wasi-libc and then all of Python's
applicable tests are run. This should help weed out any issues related
to portability and provides a relatively strong assurance check that
most everything works out.

The changes here are:

* WASIp3 is added to the Python testing matrix
* The wasi-sdk version used when testing Python is updated (keeping
up-to-date).
* The management of `wasmtime` is moved to CMake so CI doesn't install a
different version than CMake testing.
* Configuration of the runner (wasmtime version) that Python uses is
moved to CMake instead of inheriting Python's defaults. This reduces the
size of the custom patch that's carried here to test Python with.
* The custom patch is updated to adjust a test that fails on WASIp2 with
an updated version of Wasmtime. The change itself is unrelated to
wasi-libc and has to do with Wasmtime's defaults, so for now it's just
about getting CI passing.

Author: alexcrichton

.github/workflows/main.yml

@@ -294,20 +294,18 @@ jobs:
             args: -DTARGET_TRIPLE=wasm32-wasip1 -DMALLOC=emmalloc
           - name: wasm32-wasip2
             args: -DTARGET_TRIPLE=wasm32-wasip2
+          - name: wasm32-wasip3
+            args: -DTARGET_TRIPLE=wasm32-wasip3
     steps:
     - uses: actions/checkout@v6
       with:
         submodules: true
     - run: |
-        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-arm64-linux.tar.gz -L | tar xzvf -
-        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-25.0-arm64-linux" >> $GITHUB_ENV
+        curl https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-32/wasi-sdk-32.0-arm64-linux.tar.gz -L | tar xzvf -
+        echo "WASI_SDK_PATH=`pwd`/wasi-sdk-32.0-arm64-linux" >> $GITHUB_ENV
       if: runner.os == 'Linux'
       shell: bash
       working-directory: ${{ runner.tool_cache }}
-    - name: Setup `wasmtime`
-      uses: bytecodealliance/actions/wasmtime/setup@v1
-      with:
-        version: "40.0.2"
     - name: Setup wasi-libc
       run: |
         cmake -S . -B build -G Ninja \

cmake/builtins.cmake

@@ -40,11 +40,8 @@ cmake_path(GET builtins_lib_path PARENT_PATH builtins_lib_dir)
 
 if(BUILTINS_LIB)
   message(STATUS "Using builtins lib: ${BUILTINS_LIB}")
-  add_custom_command(
-    OUTPUT ${builtins_lib_path}
-    COMMAND ${CMAKE_COMMAND} -E copy ${BUILTINS_LIB} ${builtins_lib_path}
-    DEPENDS ${BUILTINS_LIB}
-  )
+  set(builtins_lib_src ${BUILTINS_LIB})
+  set(builtins_lib_dep ${BUILTINS_LIB})
 else()
   message(STATUS "Using historical builtins lib from wasi-sdk...")
   include(ExternalProject)
@@ -57,11 +54,24 @@ else()
     INSTALL_COMMAND ""
   )
   ExternalProject_Get_Property(wasi-sdk-builtins SOURCE_DIR)
-  set(src ${SOURCE_DIR}/libclang_rt.builtins-wasm32.a)
-  add_custom_command(
-    OUTPUT ${builtins_lib_path}
-    COMMAND ${CMAKE_COMMAND} -E copy ${src} ${builtins_lib_path}
-    DEPENDS wasi-sdk-builtins
-  )
+  set(builtins_lib_src ${SOURCE_DIR}/libclang_rt.builtins-wasm32.a)
+  set(builtins_lib_dep wasi-sdk-builtins)
 endif()
+
+add_custom_command(
+  OUTPUT ${builtins_lib_path}
+
+  # Copy `libclang_rt.*` into place from the source into the final destination
+  # within `tmp_resource_dir`. This is needed during linking to find
+  # `libclang_rt.*` under the appropriate path for this target.
+  COMMAND ${CMAKE_COMMAND} -E copy ${builtins_lib_src} ${builtins_lib_path}
+
+  # Additionally fill in the `include` directory for the `tmp_resource_dir` that
+  # we're creating. This is required to get access to the compiler's `stddef.h`,
+  # for example. Here we symlink to whatever the compiler already has as that's
+  # as good as we're going to get.
+  COMMAND ${CMAKE_COMMAND} -E create_symlink ${system_resource_dir}/include ${tmp_resource_dir}/include
+
+  DEPENDS ${builtins_lib_dep}
+)
 add_custom_target(builtins DEPENDS ${builtins_lib_path})

test/CMakeLists.txt

@@ -509,7 +509,8 @@ if (PYTHON_TESTS)
   find_program(PYTHON python3 python REQUIRED)
   find_program(MAKE make REQUIRED)
 
-  set(flags "--target=${TARGET_TRIPLE} --sysroot=${SYSROOT}")
+  set(cflags "--target=${TARGET_TRIPLE} --sysroot=${SYSROOT}")
+  set(ldflags "${cflags} -resource-dir ${tmp_resource_dir}")
 
   ExternalProject_Add(
     python
@@ -554,23 +555,39 @@ if (PYTHON_TESTS)
 
     BUILD_COMMAND
       ${CMAKE_COMMAND}
-        -E env CFLAGS=${flags} LDFLAGS=${flags} --
-        ${PYTHON} <SOURCE_DIR>/Tools/wasm/wasi configure-host -- --config-cache
+        -E env CFLAGS=${cflags} LDFLAGS=${ldflags} --
+        ${PYTHON} <SOURCE_DIR>/Tools/wasm/wasi configure-host
+        # Note that this gives double the stack that Python does by default to
+        # allow for minor changes in what seems to be a combination of either
+        # wasi-libc or wasmtime or something like that. Additionally, while
+        # wasip3 is being developed, this passes extra flags to enable running
+        # wasip3 binaries.
+        #
+        # Finally, the `--env` part here is copied from Python's own configuration,
+        # and if Python is updated we'll have to update this too.
+        --host-runner "\
+          ${ENGINE} run \
+            --wasm max-wasm-stack=33554432,component-model-async \
+            --wasi p3 \
+            --dir <SOURCE_DIR>::/ \
+            --env PYTHONPATH=/cross-build/wasm32-wasip1/build/lib.wasi-wasm32-3.14 \
+          "
+          -- --config-cache
     COMMAND
       ${CMAKE_COMMAND}
-        -E env CFLAGS=${flags} LDFLAGS=${flags} --
+        -E env CFLAGS=${cflags} LDFLAGS=${ldflags} --
         ${PYTHON} <SOURCE_DIR>/Tools/wasm/wasi make-host
     COMMAND
       ${CMAKE_COMMAND}
-        -E env CFLAGS=${flags} LDFLAGS=${flags} --
+        -E env CFLAGS=${cflags} LDFLAGS=${ldflags} --
         ${MAKE} --directory cross-build/wasm32-wasip1 pythoninfo
 
     INSTALL_COMMAND ""
 
     TEST_COMMAND
       ${CMAKE_COMMAND}
-        -E env CFLAGS=${flags} LDFLAGS=${flags} --
+        -E env CFLAGS=${cflags} LDFLAGS=${ldflags} --
         ${MAKE} --directory cross-build/wasm32-wasip1 test
   )
-  ExternalProject_Add_StepDependencies(python configure sysroot)
+  ExternalProject_Add_StepDependencies(python configure sysroot engine)
 endif()

test/scripts/cpython3.14.patch

@@ -10,6 +10,19 @@ index 3eac119bf8e..6a4df4bf75f 100644
        (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)))
  #  define Py_CAN_START_THREADS 1
  #endif
+diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py
+index 029c903e01a..2cdf353eebe 100644
+--- a/Lib/test/test_file.py
++++ b/Lib/test/test_file.py
+@@ -224,7 +224,7 @@ def testDefaultBufferSize(self):
+         with self.open(TESTFN, 'rb') as f:
+             data = f.read1()
+             expected_size = max(min(blksize, 8192 * 1024), io.DEFAULT_BUFFER_SIZE)
+-            self.assertEqual(len(data), expected_size)
++            self.assertTrue(len(data) <= expected_size)
+ 
+     def testTruncateOnWindows(self):
+         # SF bug <https://bugs.python.org/issue801631>
 diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
 index e2117b9588d..1a14c758c55 100644
 --- a/Lib/test/test_sys.py
@@ -23,29 +36,3 @@ index e2117b9588d..1a14c758c55 100644
 
      @unittest.skipUnless(support.is_emscripten, "only available on Emscripten")
      def test_emscripten_info(self):
-diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py
-index 54ccc95157d..e92e7419086 100644
---- a/Tools/wasm/wasi/__main__.py
-+++ b/Tools/wasm/wasi/__main__.py
-@@ -307,7 +307,7 @@ def main():
-                         # Make sure the stack size will work for a pydebug
-                         # build.
-                         # Use 16 MiB stack.
--                        "--wasm max-wasm-stack=16777216 "
-+                        "--wasm max-wasm-stack=33554432 "
-                         # Enable thread support; causes use of preview1.
-                         #"--wasm threads=y --wasi threads=y "
-                         # Map the checkout to / to load the stdlib from /Lib.
-diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
-index bcb80212362..95e4d91211e 100755
---- a/Tools/wasm/wasm_build.py
-+++ b/Tools/wasm/wasm_build.py
-@@ -329,7 +329,7 @@ def _check_wasi() -> None:
-         # workaround for https://github.com/python/cpython/issues/95952
-         "HOSTRUNNER": (
-             "wasmtime run "
--            "--wasm max-wasm-stack=16777216 "
-+            "--wasm max-wasm-stack=33554432 "
-             "--wasi preview2 "
-             "--dir {srcdir}::/ "
-             "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib"