Use wasi-sdk 20 pre-release for tests with threads (#2021)

eloparco · web-flow · commit 83659fa42af7 · 2023-03-17T20:02:03.000+08:00
`wasi-sdk-20` pre-release can be used to avoid building `wasi-libc` to enable threads.
It's not possible to use `wasi-sdk-20` pre-release on Ubuntu 20.04 because of
incompatibility with the glibc version:
```bash
/opt/wasi-sdk/bin/clang: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found
(required by /opt/wasi-sdk/bin/clang)
```
diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml
@@ -337,9 +337,13 @@ jobs:
           - os: ubuntu-20.04
             wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
             wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-19.0"
+            wasi_sysroot_option: "-DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot"
           - os: ubuntu-22.04
-            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-linux.tar.gz"
             wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-20.0+threads"
+            wasi_sysroot_option: ""
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -349,7 +353,7 @@ jobs:
           cd /opt
           sudo wget ${{ matrix.wasi_sdk_release }}
           sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv ${{ matrix.wasi_sdk_folder_name }} wasi-sdk
 
       - name: download and install wabt
         run: |
@@ -359,6 +363,7 @@ jobs:
           sudo mv wabt-1.0.31 wabt
 
       - name: build wasi-libc (needed for wasi-threads)
+        if: matrix.os == 'ubuntu-20.04'
         run: |
           mkdir wasi-libc
           cd wasi-libc
@@ -431,15 +436,22 @@ jobs:
         run: |
           cd samples/wasi-threads
           mkdir build && cd build
-          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake ${{ matrix.wasi_sysroot_option }} ..
           cmake --build . --config Release --parallel 4
           ./iwasm wasm-apps/no_pthread.wasm
 
   test:
-    needs: [build_iwasm, build_llvm_libraries_on_ubuntu_2004, build_wamrc]
-    runs-on: ubuntu-20.04
+    needs:
+      [
+        build_iwasm,
+        build_llvm_libraries_on_ubuntu_2004,
+        build_llvm_libraries_on_ubuntu_2204,
+        build_wamrc
+      ]
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-20.04, ubuntu-22.04]
         running_mode:
           [
             "classic-interp",
@@ -457,12 +469,21 @@ jobs:
             $THREADS_TEST_OPTIONS,
             $WASI_TEST_OPTIONS,
           ]
-        wasi_sdk_release:
-          [
-            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz",
-          ]
-        llvm_cache_key:
-          ["${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}"]
+        include:
+          - os: ubuntu-20.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz"
+            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-19.0"
+            wasi_sysroot_option: "WASI_SYSROOT_OPTION='--sysroot ../../../../../core/deps/wasi-libc/sysroot'"
+            ubuntu_version: "20.04"
+          - os: ubuntu-22.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
+            wasi_sdk_release: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-linux.tar.gz"
+            wabt_release: "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
+            wasi_sdk_folder_name: "wasi-sdk-20.0+threads"
+            wasi_sysroot_option: ""
+            ubuntu_version: "22.04"
         exclude:
           # uncompatiable modes and features
           # classic-interp and fast-interp don't support simd
@@ -503,10 +524,10 @@ jobs:
           cd /opt
           sudo wget ${{ matrix.wasi_sdk_release }}
           sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv ${{ matrix.wasi_sdk_folder_name }} wasi-sdk
 
       - name: build wasi-libc (needed for wasi-threads)
-        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        if: matrix.os == 'ubuntu-20.04' && matrix.test_option == '$WASI_TEST_OPTIONS'
         run: |
           mkdir wasi-libc
           cd wasi-libc
@@ -557,12 +578,12 @@ jobs:
 
       - name: Build WASI thread tests
         if: matrix.test_option == '$WASI_TEST_OPTIONS'
-        run: WASI_SYSROOT=../../../../../core/deps/wasi-libc/sysroot bash build.sh
+        run: ${{ matrix.wasi_sysroot_option }} bash build.sh
         working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/
 
       - name: build socket api tests
         if: matrix.test_option == '$WASI_TEST_OPTIONS'
-        run: WASI_SYSROOT=../../../../../core/deps/wasi-libc/sysroot bash build.sh
+        run: ${{ matrix.wasi_sysroot_option }} bash build.sh
         working-directory: ./core/iwasm/libraries/lib-socket/test/
 
       - name: run tests
@@ -577,7 +598,7 @@ jobs:
           # Add another apt repository as some packages cannot
           # be downloaded with the github default repository
           sudo curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc &&
-          sudo apt-add-repository https://packages.microsoft.com/ubuntu/20.04/prod &&
+          sudo apt-add-repository https://packages.microsoft.com/ubuntu/${{ matrix.ubuntu_version }}/prod &&
           sudo apt-get update &&
           sudo apt install -y g++-multilib lib32gcc-9-dev
 
diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml
@@ -249,7 +249,7 @@ jobs:
         os: [macos-latest]
         wasi_sdk_release:
           [
-            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-macos.tar.gz",
+            "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-macos.tar.gz",
           ]
         wabt_release:
           [
@@ -264,7 +264,7 @@ jobs:
           cd /opt
           sudo wget ${{ matrix.wasi_sdk_release }}
           sudo tar -xzf wasi-sdk-*.tar.gz
-          sudo mv wasi-sdk-19.0 wasi-sdk
+          sudo mv wasi-sdk-20.0+threads wasi-sdk
 
       - name: download and install wabt
         run: |
@@ -273,22 +273,6 @@ jobs:
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
 
-      - name: build wasi-libc (needed for wasi-threads)
-        run: |
-          mkdir wasi-libc
-          cd wasi-libc
-          git init
-          # "Rename thread_spawn import" commit on main branch
-          git fetch https://github.com/WebAssembly/wasi-libc \
-            8f5275796a82f8ecfd0833a4f3f444fa37ed4546
-          git checkout FETCH_HEAD
-          make \
-            AR=/opt/wasi-sdk/bin/llvm-ar \
-            NM=/opt/wasi-sdk/bin/llvm-nm \
-            CC=/opt/wasi-sdk/bin/clang \
-            THREAD_MODEL=posix
-        working-directory: core/deps
-
       - name: Build Sample [basic]
         run: |
           cd samples/basic
@@ -339,6 +323,6 @@ jobs:
         run: |
           cd samples/wasi-threads
           mkdir build && cd build
-          cmake -DWASI_SYSROOT=`pwd`/../../../core/deps/wasi-libc/sysroot ..
+          cmake ..
           cmake --build . --config Release --parallel 4
           ./iwasm wasm-apps/no_pthread.wasm
diff --git a/core/iwasm/libraries/lib-socket/test/build.sh b/core/iwasm/libraries/lib-socket/test/build.sh
@@ -3,18 +3,17 @@
 # Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-set -ueo pipefail
+set -eo pipefail
 CC="${CC:=/opt/wasi-sdk/bin/clang}"
 files=("tcp_udp.c" "nslookup.c")
-WASI_SYSROOT=${WASI_SYSROOT:=~/dev/wasi-libc/sysroot}
 
 for file in "${files[@]}"
 do
     echo $file
     $CC \
+        $WASI_SYSROOT_OPTION \
         --target=wasm32-wasi-threads \
         -I../inc \
-        --sysroot $WASI_SYSROOT \
         ../src/wasi/wasi_socket_ext.c -pthread -ftls-model=local-exec \
         -Wl,--allow-undefined \
         -Wl,--strip-all,--no-entry \
diff --git a/core/iwasm/libraries/lib-wasi-threads/test/build.sh b/core/iwasm/libraries/lib-wasi-threads/test/build.sh
@@ -5,16 +5,16 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 
+set -eo pipefail
 CC=${CC:=/opt/wasi-sdk/bin/clang}
-WASI_SYSROOT=${WASI_SYSROOT:=~/dev/wasi-libc/sysroot}
 WAMR_DIR=../../../../..
 
 for test_c in *.c; do
     test_wasm="$(basename $test_c .c).wasm"
 
     echo "Compiling $test_c to $test_wasm"
     $CC \
-        --sysroot $WASI_SYSROOT \
+        $WASI_SYSROOT_OPTION \
         -target wasm32-wasi-threads \
         -pthread -ftls-model=local-exec \
         -z stack-size=32768 \
diff --git a/samples/wasi-threads/CMakeLists.txt b/samples/wasi-threads/CMakeLists.txt
@@ -7,13 +7,6 @@ include(CheckPIESupported)
 
 project(wasi_threads_sample)
 
-if (NOT DEFINED WASI_SYSROOT)
-  message (WARNING "Custom sysroot with threads enabled is required to build wasi threads samples.
-Please note that current wasi-sdk doesn't ship with threads enabled.
-Run cmake command with -DWASI_SYSROOT=/path/to/sysroot/with/threads to compile samples.")
-  return ()
-endif ()
-
 ################  runtime settings  ################
 string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM)
 if (APPLE)
diff --git a/samples/wasi-threads/README.md b/samples/wasi-threads/README.md
@@ -1,21 +1,13 @@
 # "WASI threads" sample introduction
 
-Currently, since the `wasi-sdk` does not have thread support in the latest release, make sure to have [wasi-libc](https://github.com/WebAssembly/wasi-libc) installed. Build it with threads enabled, e.g.
-
-```shell
-make \
-    AR=/opt/wasi-sdk/bin/llvm-ar \
-    NM=/opt/wasi-sdk/bin/llvm-nm \
-    CC=/opt/wasi-sdk/bin/clang \
-    THREAD_MODEL=posix
-```
+To run the sample, `wasi-sdk` >= 20 is required.
 
 ## Build and run the samples
 
 ```shell
 $ mkdir build
 $ cd build
-$ cmake -DWASI_SYSROOT=/path/to/wasi-libc/sysroot ..
+$ cmake ..
 $ make
 ...
 $ ./iwasm wasm-apps/no_pthread.wasm
diff --git a/samples/wasi-threads/wasm-apps/CMakeLists.txt b/samples/wasi-threads/wasm-apps/CMakeLists.txt
@@ -11,7 +11,10 @@ if (NOT DEFINED WASI_SDK_DIR)
   set (WASI_SDK_DIR               "/opt/wasi-sdk")
 endif ()
 
-set (CMAKE_SYSROOT                  "${WASI_SYSROOT}")
+if (DEFINED WASI_SYSROOT)
+  set (CMAKE_SYSROOT                "${WASI_SYSROOT}")
+endif ()
+
 set (CMAKE_C_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_ASM_COMPILER               "${WASI_SDK_DIR}/bin/clang")
 set (CMAKE_EXE_LINKER_FLAGS         "-target wasm32-wasi-threads")
