Use libtest-mimic for wit-component tests (#1542)

* Use libtest-mimic for wit-component tests

Makes it easier to run individual tests and provides better output of
the test binaries as well.

* Fix tests on wasm

Author: alexcrichton

Cargo.lock

@@ -36,6 +36,7 @@ glob = "0.3.0"
 pretty_assertions = "1.3.0"
 env_logger = { workspace = true }
 wat = { workspace = true }
+libtest-mimic = { workspace = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
 wasmtime = { workspace = true }
@@ -44,3 +45,11 @@ wasmtime = { workspace = true }
 dummy-module = ['dep:wat']
 wat = ['dep:wast', 'dep:wat']
 semver-check = ['dummy-module']
+
+[[test]]
+name = "components"
+harness = false
+
+[[test]]
+name = "interfaces"
+harness = false

crates/wit-component/Cargo.toml

@@ -1,4 +1,5 @@
 use anyhow::{bail, Context, Error, Result};
+use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::assert_eq;
 use std::{borrow::Cow, fs, path::Path};
 use wasm_encoder::{Encode, Section};
@@ -50,137 +51,145 @@ use wit_parser::{PackageId, Resolve, UnresolvedPackage};
 ///
 /// Run the test with the environment variable `BLESS` set to update
 /// either `component.wat` or `error.txt` depending on the outcome of the encoding.
-#[test]
-fn component_encoding_via_flags() -> Result<()> {
+fn main() -> Result<()> {
     drop(env_logger::try_init());
 
+    let mut trials = Vec::new();
     for entry in fs::read_dir("tests/components")? {
         let path = entry?.path();
         if !path.is_dir() {
             continue;
         }
 
-        let test_case = path.file_stem().unwrap().to_str().unwrap();
-        println!("testing {test_case}");
+        trials.push(Trial::test(path.to_str().unwrap().to_string(), move || {
+            run_test(&path).map_err(|e| format!("{e:?}").into())
+        }));
+    }
 
-        let mut resolve = Resolve::default();
-        let (pkg, _) = resolve.push_dir(&path)?;
+    let mut args = Arguments::from_args();
+    if cfg!(target_family = "wasm") && !cfg!(target_feature = "atomics") {
+        args.test_threads = Some(1);
+    }
+    libtest_mimic::run(&args, trials).exit();
+}
 
-        let module_path = path.join("module.wat");
-        let mut adapters = glob::glob(path.join("adapt-*.wat").to_str().unwrap())?;
-        let result = if module_path.is_file() {
-            let module = read_core_module(&module_path, &resolve, pkg)?;
-            adapters
-                .try_fold(
-                    ComponentEncoder::default().module(&module)?.validate(true),
-                    |encoder, path| {
-                        let (name, wasm) = read_name_and_module("adapt-", &path?, &resolve, pkg)?;
-                        Ok::<_, Error>(encoder.adapter(&name, &wasm)?)
-                    },
-                )?
-                .encode()
-        } else {
-            let mut libs = glob::glob(path.join("lib-*.wat").to_str().unwrap())?
-                .map(|path| Ok(("lib-", path?, false)))
-                .chain(
-                    glob::glob(path.join("dlopen-lib-*.wat").to_str().unwrap())?
-                        .map(|path| Ok(("dlopen-lib-", path?, true))),
-                )
-                .collect::<Result<Vec<_>>>()?;
+fn run_test(path: &Path) -> Result<()> {
+    let test_case = path.file_stem().unwrap().to_str().unwrap();
 
-            // Sort list to ensure deterministic order, which determines priority in cases of duplicate symbols:
-            libs.sort_by(|(_, a, _), (_, b, _)| a.cmp(b));
+    let mut resolve = Resolve::default();
+    let (pkg, _) = resolve.push_dir(&path)?;
 
-            let mut linker = Linker::default().validate(true);
+    let module_path = path.join("module.wat");
+    let mut adapters = glob::glob(path.join("adapt-*.wat").to_str().unwrap())?;
+    let result = if module_path.is_file() {
+        let module = read_core_module(&module_path, &resolve, pkg)?;
+        adapters
+            .try_fold(
+                ComponentEncoder::default().module(&module)?.validate(true),
+                |encoder, path| {
+                    let (name, wasm) = read_name_and_module("adapt-", &path?, &resolve, pkg)?;
+                    Ok::<_, Error>(encoder.adapter(&name, &wasm)?)
+                },
+            )?
+            .encode()
+    } else {
+        let mut libs = glob::glob(path.join("lib-*.wat").to_str().unwrap())?
+            .map(|path| Ok(("lib-", path?, false)))
+            .chain(
+                glob::glob(path.join("dlopen-lib-*.wat").to_str().unwrap())?
+                    .map(|path| Ok(("dlopen-lib-", path?, true))),
+            )
+            .collect::<Result<Vec<_>>>()?;
 
-            if path.join("stub-missing-functions").is_file() {
-                linker = linker.stub_missing_functions(true);
-            }
+        // Sort list to ensure deterministic order, which determines priority in cases of duplicate symbols:
+        libs.sort_by(|(_, a, _), (_, b, _)| a.cmp(b));
 
-            if path.join("use-built-in-libdl").is_file() {
-                linker = linker.use_built_in_libdl(true);
-            }
+        let mut linker = Linker::default().validate(true);
 
-            let linker =
-                libs.into_iter()
-                    .try_fold(linker, |linker, (prefix, path, dl_openable)| {
-                        let (name, wasm) = read_name_and_module(prefix, &path, &resolve, pkg)?;
-                        Ok::<_, Error>(linker.library(&name, &wasm, dl_openable)?)
-                    })?;
+        if path.join("stub-missing-functions").is_file() {
+            linker = linker.stub_missing_functions(true);
+        }
 
-            adapters
-                .try_fold(linker, |linker, path| {
-                    let (name, wasm) = read_name_and_module("adapt-", &path?, &resolve, pkg)?;
-                    Ok::<_, Error>(linker.adapter(&name, &wasm)?)
-                })?
-                .encode()
-        };
-        let component_path = path.join("component.wat");
-        let component_wit_path = path.join("component.wit.print");
-        let error_path = path.join("error.txt");
+        if path.join("use-built-in-libdl").is_file() {
+            linker = linker.use_built_in_libdl(true);
+        }
 
-        let bytes = match result {
-            Ok(bytes) => {
-                if test_case.starts_with("error-") {
-                    bail!("expected an error but got success");
-                }
-                bytes
+        let linker = libs
+            .into_iter()
+            .try_fold(linker, |linker, (prefix, path, dl_openable)| {
+                let (name, wasm) = read_name_and_module(prefix, &path, &resolve, pkg)?;
+                Ok::<_, Error>(linker.library(&name, &wasm, dl_openable)?)
+            })?;
+
+        adapters
+            .try_fold(linker, |linker, path| {
+                let (name, wasm) = read_name_and_module("adapt-", &path?, &resolve, pkg)?;
+                Ok::<_, Error>(linker.adapter(&name, &wasm)?)
+            })?
+            .encode()
+    };
+    let component_path = path.join("component.wat");
+    let component_wit_path = path.join("component.wit.print");
+    let error_path = path.join("error.txt");
+
+    let bytes = match result {
+        Ok(bytes) => {
+            if test_case.starts_with("error-") {
+                bail!("expected an error but got success");
             }
-            Err(err) => {
-                if !test_case.starts_with("error-") {
-                    return Err(err);
-                }
-                assert_output(&format!("{err:?}"), &error_path)?;
-                continue;
+            bytes
+        }
+        Err(err) => {
+            if !test_case.starts_with("error-") {
+                return Err(err);
             }
-        };
+            assert_output(&format!("{err:?}"), &error_path)?;
+            return Ok(());
+        }
+    };
 
-        let wat = wasmprinter::print_bytes(&bytes)?;
-        assert_output(&wat, &component_path)?;
-        let (pkg, resolve) = match wit_component::decode(&bytes)? {
-            DecodedWasm::WitPackage(..) => unreachable!(),
-            DecodedWasm::Component(resolve, world) => {
-                (resolve.worlds[world].package.unwrap(), resolve)
-            }
-        };
-        let wit = WitPrinter::default().print(&resolve, pkg)?;
-        assert_output(&wit, &component_wit_path)?;
+    let wat = wasmprinter::print_bytes(&bytes)?;
+    assert_output(&wat, &component_path)?;
+    let (pkg, resolve) = match wit_component::decode(&bytes)? {
+        DecodedWasm::WitPackage(..) => unreachable!(),
+        DecodedWasm::Component(resolve, world) => (resolve.worlds[world].package.unwrap(), resolve),
+    };
+    let wit = WitPrinter::default().print(&resolve, pkg)?;
+    assert_output(&wit, &component_wit_path)?;
 
-        UnresolvedPackage::parse(&component_wit_path, &wit)
-            .context("failed to parse printed WIT")?;
+    UnresolvedPackage::parse(&component_wit_path, &wit).context("failed to parse printed WIT")?;
 
-        // Check that the producer data got piped through properly
-        let metadata = wasm_metadata::Metadata::from_binary(&bytes)?;
-        match metadata {
-            // Depends on the ComponentEncoder always putting the first module as the 0th child:
-            wasm_metadata::Metadata::Component { children, .. } => match children[0].as_ref() {
-                wasm_metadata::Metadata::Module { producers, .. } => {
-                    let producers = producers.as_ref().expect("child module has producers");
-                    let processed_by = producers
-                        .get("processed-by")
-                        .expect("child has processed-by section");
+    // Check that the producer data got piped through properly
+    let metadata = wasm_metadata::Metadata::from_binary(&bytes)?;
+    match metadata {
+        // Depends on the ComponentEncoder always putting the first module as the 0th child:
+        wasm_metadata::Metadata::Component { children, .. } => match children[0].as_ref() {
+            wasm_metadata::Metadata::Module { producers, .. } => {
+                let producers = producers.as_ref().expect("child module has producers");
+                let processed_by = producers
+                    .get("processed-by")
+                    .expect("child has processed-by section");
+                assert_eq!(
+                    processed_by
+                        .get("wit-component")
+                        .expect("wit-component producer present"),
+                    env!("CARGO_PKG_VERSION")
+                );
+                if module_path.is_file() {
                     assert_eq!(
                         processed_by
-                            .get("wit-component")
-                            .expect("wit-component producer present"),
-                        env!("CARGO_PKG_VERSION")
+                            .get("my-fake-bindgen")
+                            .expect("added bindgen field present"),
+                        "123.45"
                     );
-                    if module_path.is_file() {
-                        assert_eq!(
-                            processed_by
-                                .get("my-fake-bindgen")
-                                .expect("added bindgen field present"),
-                            "123.45"
-                        );
-                    } else {
-                        // Otherwise, we used `Linker`, which synthesizes the
-                        // "main" module and thus won't have `my-fake-bindgen`
-                    }
+                } else {
+                    // Otherwise, we used `Linker`, which synthesizes the
+                    // "main" module and thus won't have `my-fake-bindgen`
                 }
-                _ => panic!("expected child to be a module"),
-            },
-            _ => panic!("expected top level metadata of component"),
-        }
+            }
+            _ => panic!("expected child to be a module"),
+        },
+        _ => panic!("expected top level metadata of component"),
     }
 
     Ok(())

crates/wit-component/tests/components.rs

@@ -1,4 +1,5 @@
 use anyhow::{Context, Result};
+use libtest_mimic::{Arguments, Trial};
 use pretty_assertions::assert_eq;
 use std::fs;
 use std::path::Path;
@@ -17,10 +18,10 @@ use wit_parser::{PackageId, Resolve, UnresolvedPackage};
 ///
 /// Run the test with the environment variable `BLESS` set to update
 /// the baseline files.
-#[test]
-fn interface_encoding() -> Result<()> {
+fn main() -> Result<()> {
     env_logger::init();
 
+    let mut trials = Vec::new();
     for entry in fs::read_dir("tests/interfaces")? {
         let path = entry?.path();
         let name = match path.file_name().and_then(|s| s.to_str()) {
@@ -30,15 +31,22 @@ fn interface_encoding() -> Result<()> {
         let is_dir = path.is_dir();
         let is_test = is_dir || name.ends_with(".wit");
         if is_test {
-            run_test(&path, is_dir).context(format!("failed test `{}`", path.display()))?;
+            trials.push(Trial::test(name.to_string(), move || {
+                run_test(&path, is_dir)
+                    .context(format!("failed test `{}`", path.display()))
+                    .map_err(|e| format!("{e:?}").into())
+            }));
         }
     }
 
-    Ok(())
+    let mut args = Arguments::from_args();
+    if cfg!(target_family = "wasm") && !cfg!(target_feature = "atomics") {
+        args.test_threads = Some(1);
+    }
+    libtest_mimic::run(&args, trials).exit();
 }
 
 fn run_test(path: &Path, is_dir: bool) -> Result<()> {
-    println!("running test at {path:?}");
     let mut resolve = Resolve::new();
     let package = if is_dir {
         resolve.push_dir(path)?.0