Refactor the CMake and related scripts for unit tests (#4605)

- only unit/CMakeLists.txt fetches googletest
- only unit/unit_common.cmake find LLVM dependencies
- use `WAMR_RUNTIME_LIB_SOURCE` to replace a long list
- enable LLVM support across various CMake configurations
- remove redundant LLVM dependency installation and add i386 support for libraries
- CMake configurations for better build output and module support
- Find llvm on demand
- Add test suite guidelines

Author: lum1n0us

.github/workflows/compilation_on_android_ubuntu.yml

@@ -350,7 +350,7 @@ jobs:
         uses: ./.github/actions/install-wasi-sdk-wabt
         with:
           os: ${{ matrix.os }}
-
+      
       - name: Build wamrc
         run: |
           mkdir build && cd build
@@ -361,15 +361,16 @@ jobs:
       - name: Install dependencies for X86_32
         if: matrix.build_target == 'X86_32'
         run: |
+          sudo dpkg --add-architecture i386
           sudo apt-get update
-          sudo apt-get install -y g++-multilib
+          sudo apt-get install -y g++-multilib libzstd-dev:i386 zlib1g-dev:i386
 
       - name: Build and run unit tests
         run: |
           mkdir build && cd build
           cmake .. -DWAMR_BUILD_TARGET=${{ matrix.build_target }}
-          cmake --build . --config Release --parallel 4
-          ctest
+          cmake --build . --parallel 4
+          ctest --output-on-failure
         working-directory: tests/unit
 
   build_regression_tests:

build-scripts/code_coverage.cmake

@@ -0,0 +1,37 @@
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+function(check_ubuntu_version)
+# ubuntu 2204 is the recommended environment for collecting coverage data for now
+# otherwise, there will be ERRORs, when using 2404, like below and
+#
+# geninfo: ERROR: ('mismatch') mismatched end line for _ZN63compilation_aot_emit_memory_test_aot_check_memory_overflow_Test8TestBodyEv at /workspaces/wasm-micro-runtime/tests/unit/compilation/aot_emit_memory_test.cc:96: 96 -> 106
+#        (use "geninfo --ignore-errors mismatch,mismatch ..." to suppress this warning)
+# geninfo: ERROR: ('negative') Unexpected negative count '-3' for /workspaces/wasm-micro-runtime/core/iwasm/interpreter/wasm_interp_classic.c:5473.
+#        Perhaps you need to compile with '-fprofile-update=atomic
+#        (use "geninfo --ignore-errors negative,negative ..." to suppress this warning)
+#
+# For sure, `--ignore-errors` can be used to ignore these errors, but better to use the recommended environment.
+    file(READ "/etc/os-release" OS_RELEASE_CONTENT)
+    string(REGEX MATCH "VERSION_ID=\"([0-9]+)\\.([0-9]+)\"" _ ${OS_RELEASE_CONTENT})
+    if(NOT DEFINED CMAKE_MATCH_1 OR NOT DEFINED CMAKE_MATCH_2)
+        message(WARNING "Unable to detect Ubuntu version. Please ensure you are using Ubuntu 22.04.")
+        return()
+    endif()
+
+    set(UBUNTU_MAJOR_VERSION ${CMAKE_MATCH_1})
+    set(UBUNTU_MINOR_VERSION ${CMAKE_MATCH_2})
+
+    if(NOT (UBUNTU_MAJOR_VERSION EQUAL 22 AND UBUNTU_MINOR_VERSION EQUAL 04))
+        message(WARNING "Ubuntu ${UBUNTU_MAJOR_VERSION}.${UBUNTU_MINOR_VERSION} detected. Ubuntu 22.04 is recommended for collecting coverage data.")
+    else()
+        message(STATUS "Ubuntu 22.04 detected. Proceeding with coverage data collection.")
+    endif()
+endfunction()
+
+check_ubuntu_version()
+
+# add compile options for code coverage globally
+add_compile_options(--coverage -O0 -g)
+link_libraries(gcov)
+add_definitions (-DCOLLECT_CODE_COVERAGE)

build-scripts/config_common.cmake

@@ -640,9 +640,7 @@ if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1)
   message ("     GC heap verification enabled")
 endif ()
 if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)
-  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
-  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
-  add_definitions (-DCOLLECT_CODE_COVERAGE)
+  include(${CMAKE_CURRENT_LIST_DIR}/code_coverage.cmake)
   message ("     Collect code coverage enabled")
 endif ()
 if (WAMR_BUILD_STATIC_PGO EQUAL 1)

tests/unit/CMakeLists.txt

@@ -30,13 +30,8 @@ if(WAMR_BUILD_TARGET STREQUAL "X86_32")
   set(CMAKE_LIBRARY_ARCHITECTURE "i386-linux-gnu" CACHE STRING "" FORCE)
 endif()
 
-# Prevent overriding the parent project's compiler/linker
-# settings on Windows
-set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
-
 # Fetch Google test
 include (FetchContent)
-
 if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24")
   FetchContent_Declare (
     googletest
@@ -50,10 +45,11 @@ else()
   )
 endif()
 
+# Prevent overriding the parent project's compiler/linker
+# settings on Windows
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 FetchContent_MakeAvailable(googletest)
 
-SET(GOOGLETEST_INCLUDED 1)
-
 include(GoogleTest)
 enable_testing()
 
@@ -64,12 +60,13 @@ add_subdirectory(libc-builtin)
 add_subdirectory(shared-utils)
 add_subdirectory(linear-memory-wasm)
 add_subdirectory(linear-memory-aot)
-add_subdirectory(aot-stack-frame)
 add_subdirectory(linux-perf)
 add_subdirectory(gc)
 add_subdirectory(tid-allocator)
 
 if (NOT WAMR_BUILD_TARGET STREQUAL "X86_32")
+  add_subdirectory(aot-stack-frame)
+
   # should enable 32-bit llvm when X86_32
   add_subdirectory (aot)
   add_subdirectory (custom-section)

tests/unit/README.md

@@ -0,0 +1,195 @@
+# Guide to Creating a Test Suite for a New Feature in WAMR
+
+This guide provides instructions for contributors on how to create a test suite for a new feature in the WAMR project. Follow these steps to ensure consistency and maintainability across the test framework.
+
+---
+
+## General Guidelines
+
+- **Create a New Directory**:
+  Always create a dedicated directory for a new feature under the `tests/unit/` directory.
+
+  - Reuse existing test cases and patch them when possible to avoid redundancy.
+  - Name the directory in lowercase with words separated by hyphens (e.g., `new-feature`).
+  - Name the test source file in lowercase with words separated by underscore (e.g., `new_test.cc`).
+
+- **Avoid Committing `.wasm` Files**:
+  Do not commit precompiled `.wasm` files. Instead:
+
+  - Generate `.wasm` files from `.wat` or `.c` source files.
+  - Use `ExternalProject` and the `wasi-sdk` toolchain to compile `.wasm` files during the build process.
+
+- **Keep Using `ctest` as the framework**:
+  Continue to use `ctest` for running the test cases, as it is already integrated into the existing test framework.
+
+---
+
+## Writing `CMakeLists.txt` for the Test Suite
+
+When creating a `CMakeLists.txt` file for your test suite, follow these best practices:
+
+1. **Do Not Fetch Googletest Again**:
+   The root `unit/CMakeLists.txt` already fetches Googletest. Avoid including or fetching it again in your test suite.
+
+2. **Find LLVM on Demand**:
+   If your test suite requires LLVM, use `find_package` to locate LLVM components as needed. Do not include LLVM globally unless required.
+
+3. **Include `unit_common.cmake`**:
+   Always include `../unit_common.cmake` in your `CMakeLists.txt` to avoid duplicating common configurations and utilities.
+
+   Example:
+
+   ```cmake
+   include("../unit_common.cmake")
+   ```
+
+4. **Use `WAMR_RUNTIME_LIB_SOURCE`**:
+   Replace long lists of runtime source files with the `WAMR_RUNTIME_LIB_SOURCE` variable to simplify your configuration.
+
+   Example:
+
+   ```cmake
+   target_sources(your_test_target PRIVATE ${WAMR_RUNTIME_LIB_SOURCE})
+   ```
+
+5. **Avoid Global Compilation Flags**:
+   Do not define global compilation flags in the `unit` directory. Each test case should specify its own compilation flags based on its unique requirements.
+
+---
+
+## Generating `.wasm` Files
+
+- **Compile `.wasm` Files Dynamically**:
+  Use `ExternalProject` in your `CMakeLists.txt` to compile `.wasm` files from `.wat` or `.c` source files.
+  - Use the `wasi-sdk` toolchain for `.c` or `.cc` source files.
+  - Example configuration:
+    ```cmake
+    ExternalProject_Add(
+        generate_wasm
+        SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps
+        BUILD_ALWAYS YES
+        CONFIGURE_COMMAND  ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build
+                              -DWASI_SDK_PREFIX=${WASI_SDK_DIR}
+                              -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN}
+        BUILD_COMMAND      ${CMAKE_COMMAND} --build build
+        INSTALL_COMMAND    ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR}/wasm-apps
+    )
+    ```
+- **Example for `wasm-apps` Directory**:
+  Place your source files in a `wasm-apps/` subdirectory within your test suite directory.
+
+  - Create a `CMakeLists.txt` in `wasm-apps/` to handle the compilation of these files.
+  - Example `CMakeLists.txt` for `wasm-apps/`:
+
+    ```cmake
+    cmake_minimum_required(VERSION 3.13)
+    project(wasm_apps)
+
+    add_executable(example example.c)
+    set_target_properties(example PROPERTIES SUFFIX .wasm)
+    install(TARGETS example DESTINATION .)
+    ```
+
+---
+
+## Compiling and Running Test Cases
+
+To compile and run the test cases, follow these steps:
+
+1. **Generate Build Files**:
+
+   ```bash
+   cmake -S . -B build
+   ```
+
+2. **Build the Test Suite**:
+
+   ```bash
+   cmake --build build
+   ```
+
+3. **Run the Tests**:
+
+   ```bash
+   ctest --test-dir build --output-on-failure
+   ```
+
+   This will compile and execute all test cases in the test suite, displaying detailed output for any failures.
+
+4. **List all Tests**:
+   To see all available test cases, use:
+
+   ```bash
+   ctest --test-dir build -N
+   ```
+
+5. **Run a Specific Test**:
+   To run a specific test case, use:
+   ```bash
+   ctest --test-dir build -R <test_name> --output-on-failure
+   ```
+
+---
+
+## Collecting Code Coverage Data
+
+To collect code coverage data using `lcov`, follow these steps:
+
+1. **Build with Coverage Flags**:
+   Ensure the test suite is built with coverage flags enabled:
+
+   ```bash
+   cmake -S . -B build -DCOLLECT_CODE_COVERAGE=1
+   cmake --build build
+   ```
+
+2. **Run the Tests**:
+   Execute the test cases as described above.
+
+3. **Generate Coverage Report**:
+   Use `lcov` to collect and generate the coverage report:
+
+   ```bash
+   lcov --capture --directory build --output-file coverage.all.info
+   lcov --extract coverage.all.info "*/core/iwasm/*" "*/core/shared/*" --output-file coverage.info
+   genhtml coverage.info --output-directory coverage-report
+   ```
+
+4. **View the Report**:
+   Open the `index.html` file in the `coverage-report` directory to view the coverage results in your browser.
+
+5. **Summary of Coverage**:
+   To get a summary of the coverage data, use:
+
+   ```bash
+   lcov --summary coverage.info
+   ```
+
+---
+
+## Example Directory Structure
+
+Here’s an example of how your test suite directory might look:
+
+```
+new-feature/
+├── CMakeLists.txt
+├── new_feature_test.cc
+├── wasm-apps/
+|   ├── CMakeLists.txt
+│   ├── example.c
+│   └── example.wat
+```
+
+---
+
+## Additional Notes
+
+- **Testing Framework**: Use Googletest for writing unit tests. Refer to existing test cases in the `tests/unit/` directory for examples.
+- **Documentation**: Add comments in your test code to explain the purpose of each test case.
+- **Edge Cases**: Ensure your test suite covers edge cases and potential failure scenarios.
+- **Reuse Utilities**: Leverage existing utilities in `common/` (e.g., `mock_allocator.h`, `test_helper.h`) to simplify your test code.
+
+---
+
+By following these guidelines, you can create a well-structured and maintainable test suite that integrates seamlessly with the WAMR testing framework.

tests/unit/aot-stack-frame/CMakeLists.txt

@@ -8,7 +8,7 @@ project (test-aot-stack-frame)
 add_definitions (-DRUN_ON_LINUX)
 
 set (WAMR_BUILD_AOT 1)
-set (WAMR_BUILD_INTERP 0)
+set (WAMR_BUILD_INTERP 1)
 set (WAMR_BUILD_JIT 0)
 set (WAMR_BUILD_SIMD 1)
 set (WAMR_BUILD_REF_TYPES 1)
@@ -21,6 +21,10 @@ set (WAMR_BUILD_GC 1)
 
 include (../unit_common.cmake)
 
+find_package(LLVM REQUIRED CONFIG)
+include_directories(${LLVM_INCLUDE_DIRS})
+add_definitions(${LLVM_DEFINITIONS})
+
 include_directories (${CMAKE_CURRENT_SOURCE_DIR})
 
 add_definitions (-DWASM_ENABLE_AOT_STACK_FRAME=1)
@@ -32,22 +36,10 @@ file (GLOB_RECURSE source_all ${CMAKE_CURRENT_SOURCE_DIR}/*.cc)
 set (UNIT_SOURCE ${source_all})
 
 set (unit_test_sources
-        ${UNIT_SOURCE}
-        ${WAMR_RUNTIME_LIB_SOURCE}
-        ${UNCOMMON_SHARED_SOURCE}
-        ${SRC_LIST}
-        ${PLATFORM_SHARED_SOURCE}
-        ${UTILS_SHARED_SOURCE}
-        ${MEM_ALLOC_SHARED_SOURCE}
-        ${LIB_HOST_AGENT_SOURCE}
-        ${NATIVE_INTERFACE_SOURCE}
-        ${LIBC_BUILTIN_SOURCE}
-        ${IWASM_COMMON_SOURCE}
-        ${IWASM_INTERP_SOURCE}
-        ${IWASM_AOT_SOURCE}
-        ${IWASM_COMPL_SOURCE}
-        ${WASM_APP_LIB_SOURCE_ALL}
-    )
+  ${UNIT_SOURCE}
+  ${WAMR_RUNTIME_LIB_SOURCE}
+  ${IWASM_COMPL_SOURCE}
+)
 
 # Automatically build wasm-apps for this test
 add_subdirectory(wasm-apps)
@@ -59,4 +51,4 @@ add_dependencies (aot_stack_frame_test aot-stack-frame-test-wasm)
 
 target_link_libraries (aot_stack_frame_test ${LLVM_AVAILABLE_LIBS} gtest_main )
 
-#gtest_discover_tests(aot_stack_frame_test)
+gtest_discover_tests(aot_stack_frame_test)