Refactor how string encodings are specified to wit-component (#401)

* Refactor how string encodings are specified to `wit-component`

This commit is motivated by the recent addition of utf-16 support into
the C generator. Currently that works by "passing a flag" to the
`wit-component` invocation effectively, but the level of abstraction of
this flag isn't quite right:

* The utf16 flag must be specified in two separate locations that need
  to be kept in sync.
* Interfaces aren't necessarily all generated in one location, meaning
  that it's not always correct to specify utf16 for everything all
  at-once. Instead it would be better to on a per-unit-generation
  granularity for the encoding to be specified.

The contents of this commit aim to address this issue by changing the
`component-type` custom section in wasm binaries generated by
guest-generators to include the string encoding used. The encoding of
strings wasn't moved into the `Interface` structure since it's an
implementation-level detail rather than an interface-level detail. The
`component-type` custom section is now encoded as a leading version
byte, a string encoding byte, followed by the component as before.

The `wit-component` tooling internally has been refactored to take all
of this into account. Notably the import/exports/default lists were
replaced with a new `BindgenMetadata` structure which handles all the
merging between different custom sections and CLI flags. Additionally
new maps are plumbed around which keep track of core wasm
imports/exports and their registered encoding to ensure that the
liftings/lowerings use the right encoding.

The main limitation of this commit is that the same interface cannot be
imported with two different encodings into the same core wasm module.
Implementing this will require unique names of the import into the
module (e.g. some form of name mangling). Otherwise for now it returns
an error.

It's expected that user-level interactions with `wit-component` are
largely unchanged after this commit. The main change is that if the
`wit-component` tool is run manually then the `--encoding` flag is
largely not necessary unless the binary doesn't previously have
`component-type` embedded sections.

* Fix demo build

Author: alexcrichton

crates/gen-guest-c/src/component_type_object.rs

@@ -1,16 +1,20 @@
-use anyhow::{Context, Result};
+use anyhow::Result;
 use heck::ToSnakeCase;
 use wasm_encoder::{
     CodeSection, CustomSection, Encode, Function, FunctionSection, Module, TypeSection,
 };
-use wit_component::{ComponentEncoder, ComponentInterfaces};
+use wit_component::{ComponentInterfaces, StringEncoding};
 
 pub fn linking_symbol(name: &str) -> String {
     let snake = name.to_snake_case();
     format!("__component_type_object_force_link_{snake}")
 }
 
-pub fn object(name: &str, interfaces: &ComponentInterfaces) -> Result<Vec<u8>> {
+pub fn object(
+    name: &str,
+    interfaces: &ComponentInterfaces,
+    encoding: StringEncoding,
+) -> Result<Vec<u8>> {
     let mut module = Module::new();
 
     // Build a module with one function that's a "dummy function"
@@ -24,18 +28,7 @@ pub fn object(name: &str, interfaces: &ComponentInterfaces) -> Result<Vec<u8>> {
     code.function(&Function::new([]));
     module.section(&code);
 
-    let mut encoder = ComponentEncoder::default()
-        .imports(interfaces.imports.values().cloned())?
-        .exports(interfaces.exports.values().cloned())?;
-
-    if let Some(default) = &interfaces.default {
-        encoder = encoder.interface(default.clone())?;
-    }
-
-    let data = encoder
-        .types_only(true)
-        .encode()
-        .with_context(|| format!("translating {name} to component type"))?;
+    let data = wit_component::metadata::encode(interfaces, encoding);
 
     // The custom section name here must start with "component-type" but
     // otherwise is attempted to be unique here to ensure that this doesn't get

crates/gen-guest-c/src/lib.rs

@@ -271,7 +271,7 @@ impl WorldGenerator for C {
         files.push(&format!("{snake}.h"), h_str.as_bytes());
         files.push(
             &format!("{snake}_component_type.o",),
-            component_type_object::object(name, interfaces)
+            component_type_object::object(name, interfaces, self.opts.string_encoding)
                 .unwrap()
                 .as_slice(),
         );

crates/gen-guest-rust/src/lib.rs

@@ -178,18 +178,8 @@ impl WorldGenerator for RustWasm {
         self.src
             .push_str(&format!("#[link_section = \"component-type:{name}\"]\n"));
 
-        let mut encoder = wit_component::ComponentEncoder::default()
-            .imports(interfaces.imports.values().cloned())
-            .unwrap()
-            .exports(interfaces.exports.values().cloned())
-            .unwrap();
-        if let Some(default) = &interfaces.default {
-            encoder = encoder.interface(default.clone()).unwrap();
-        }
-        let component_type = encoder
-            .types_only(true)
-            .encode()
-            .expect("encoding a component type");
+        let component_type =
+            wit_component::metadata::encode(interfaces, wit_component::StringEncoding::UTF8);
         self.src.push_str(&format!(
             "pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; {}] = ",
             component_type.len()

crates/test-helpers/macros/build.rs

@@ -1,6 +1,6 @@
 use std::env;
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use wit_bindgen_core::{wit_parser::Interface, Direction, Generator};
 use wit_component::{ComponentEncoder, ComponentInterfaces, StringEncoding};
@@ -18,19 +18,8 @@ fn guest_c(
         if !c_impl.exists() {
             continue;
         }
-        let imports = test_dir.join("imports.wit");
-        let exports = test_dir.join("exports.wit");
-        println!("cargo:rerun-if-changed={}", imports.display());
-        println!("cargo:rerun-if-changed={}", exports.display());
         println!("cargo:rerun-if-changed={}", c_impl.display());
-
-        let import = Interface::parse_file(&test_dir.join("imports.wit")).unwrap();
-        let export = Interface::parse_file(&test_dir.join("exports.wit")).unwrap();
-        let interfaces = ComponentInterfaces {
-            imports: [(import.name.clone(), import)].into_iter().collect(),
-            exports: Default::default(),
-            default: Some(export),
-        };
+        let interfaces = read_interfaces(&test_dir);
         let name = test_dir.file_name().unwrap().to_str().unwrap();
         let snake = name.replace("-", "_");
         let mut files = Default::default();
@@ -89,11 +78,7 @@ fn guest_c(
 
         // Translate the canonical ABI module into a component.
         let module = fs::read(&out_wasm).expect("failed to read wasm file");
-        let mut encoder = ComponentEncoder::default();
-        if utf_16 {
-            encoder = encoder.encoding(StringEncoding::UTF16);
-        }
-        let component = encoder
+        let component = ComponentEncoder::default()
             .module(module.as_slice())
             .expect("pull custom sections from module")
             .validate(true)
@@ -218,13 +203,13 @@ fn main() {
             drop(fs::remove_dir_all(&out_dir));
 
             let java_dir = out_dir.join("src/main/java");
+            let interfaces = read_interfaces(&test_dir);
 
             for (wit, direction) in [
                 ("imports.wit", Direction::Import),
                 ("exports.wit", Direction::Export),
             ] {
                 let path = test_dir.join(wit);
-                println!("cargo:rerun-if-changed={}", path.display());
                 let iface = Interface::parse_file(&path).unwrap();
                 let package_dir = java_dir.join(&format!("wit_{}", iface.name));
                 fs::create_dir_all(&package_dir).unwrap();
@@ -281,18 +266,13 @@ fn main() {
 
             let out_wasm = out_dir.join("target/generated/wasm/teavm-wasm/classes.wasm");
 
-            let imports = [Interface::parse_file(test_dir.join("imports.wit")).unwrap()];
-            let interface = Interface::parse_file(test_dir.join("exports.wit")).unwrap();
-
             // Translate the canonical ABI module into a component.
             // The wit interfaces are explicitly passed to ComponentEncoder,
             // because the TeaVM guest doesnt yet support putting component
             // types into custom sections.
             let module = fs::read(&out_wasm).expect("failed to read wasm file");
             let component = ComponentEncoder::default()
-                .imports(imports)
-                .unwrap()
-                .interface(interface)
+                .interfaces(interfaces, StringEncoding::UTF8)
                 .unwrap()
                 .module(module.as_slice())
                 .expect("pull custom sections from module")
@@ -321,16 +301,29 @@ fn main() {
     std::fs::write(out_dir.join("wasms.rs"), src).unwrap();
 }
 
-#[cfg(unix)]
-fn mvn() -> Command {
-    Command::new("mvn")
+fn read_interfaces(dir: &Path) -> ComponentInterfaces {
+    let imports = dir.join("imports.wit");
+    let exports = dir.join("exports.wit");
+    println!("cargo:rerun-if-changed={}", imports.display());
+    println!("cargo:rerun-if-changed={}", exports.display());
+
+    let import = Interface::parse_file(&imports).unwrap();
+    let export = Interface::parse_file(&exports).unwrap();
+    ComponentInterfaces {
+        imports: [(import.name.clone(), import)].into_iter().collect(),
+        exports: Default::default(),
+        default: Some(export),
+    }
 }
 
-#[cfg(windows)]
 fn mvn() -> Command {
-    let mut cmd = Command::new("cmd");
-    cmd.args(&["/c", "mvn"]);
-    cmd
+    if cfg!(windows) {
+        let mut cmd = Command::new("cmd");
+        cmd.args(&["/c", "mvn"]);
+        cmd
+    } else {
+        Command::new("mvn")
+    }
 }
 
 fn pom_xml(classes_to_preserve: &[&str]) -> Vec<u8> {

crates/test-helpers/src/lib.rs

@@ -145,24 +145,25 @@ pub fn run_component_codegen_test(
     generate: fn(&str, &[u8], &mut Files),
     verify: fn(&Path, &str),
 ) {
-    let mut encoder = wit_component::ComponentEncoder::default();
     let iface = Interface::parse_file(wit_path).unwrap();
+    let mut interfaces = ComponentInterfaces::default();
 
-    let wasm = match dir {
+    match dir {
         Direction::Import => {
-            encoder = encoder.imports([iface.clone()]).unwrap();
-            dummy_module(&[iface], &[], None)
+            interfaces.imports.insert(iface.name.clone(), iface);
         }
         Direction::Export => {
-            encoder = encoder.interface(iface.clone()).unwrap();
-            dummy_module(&[], &[], Some(&iface))
+            interfaces.default = Some(iface);
         }
-    };
+    }
 
-    let component = encoder
+    let wasm = dummy_module(&interfaces);
+    let component = wit_component::ComponentEncoder::default()
         .module(&wasm)
         .unwrap()
         .validate(true)
+        .interfaces(interfaces, wit_component::StringEncoding::UTF8)
+        .unwrap()
         .encode()
         .unwrap();
 
@@ -189,14 +190,10 @@ pub fn run_component_codegen_test(
     verify(&dir, name);
 }
 
-pub fn dummy_module(
-    imports: &[Interface],
-    exports: &[Interface],
-    default: Option<&Interface>,
-) -> Vec<u8> {
+pub fn dummy_module(interfaces: &ComponentInterfaces) -> Vec<u8> {
     let mut wat = String::new();
     wat.push_str("(module\n");
-    for import in imports {
+    for (_, import) in interfaces.imports.iter() {
         for func in import.functions.iter() {
             let sig = import.wasm_signature(AbiVariant::GuestImport, func);
 
@@ -210,14 +207,14 @@ pub fn dummy_module(
         }
     }
 
-    for export in exports {
+    for (_, export) in interfaces.exports.iter() {
         for func in export.functions.iter() {
             let name = format!("{}#{}", export.name, func.name);
             push_func(&mut wat, &name, export, func);
         }
     }
 
-    if let Some(default) = default {
+    if let Some(default) = &interfaces.default {
         for func in default.functions.iter() {
             push_func(&mut wat, &func.name, default, func);
         }

crates/wit-bindgen-demo/src/lib.rs

@@ -54,6 +54,19 @@ fn init() {
 
 fn render(lang: demo::Lang, wit: &str, files: &mut Files, options: &demo::Options) -> Result<()> {
     let iface = Interface::parse("input", &wit)?;
+    let interfaces = ComponentInterfaces {
+        imports: if options.import {
+            [(iface.name.clone(), iface.clone())].into_iter().collect()
+        } else {
+            Default::default()
+        },
+        exports: Default::default(),
+        default: if !options.import {
+            Some(iface.clone())
+        } else {
+            None
+        },
+    };
 
     let gen_world_legacy = |mut gen: Box<dyn Generator>, files: &mut Files| {
         let (imports, exports) = if options.import {
@@ -65,16 +78,6 @@ fn render(lang: demo::Lang, wit: &str, files: &mut Files, options: &demo::Option
     };
 
     let gen_world = |mut gen: Box<dyn WorldGenerator>, files: &mut Files| {
-        let (imports, default) = if options.import {
-            (vec![iface.clone()], None)
-        } else {
-            (vec![], Some(iface.clone()))
-        };
-        let interfaces = ComponentInterfaces {
-            imports: imports.into_iter().map(|i| (i.name.clone(), i)).collect(),
-            exports: Default::default(),
-            default,
-        };
         gen.generate("demo", &interfaces, files);
     };
 
@@ -86,19 +89,11 @@ fn render(lang: demo::Lang, wit: &str, files: &mut Files, options: &demo::Option
     // interface and dummy module.  Finally this component is fed into the host
     // generator which gives us the files we want.
     let gen_component = |mut gen: Box<dyn ComponentGenerator>, files: &mut Files| {
-        let (imports, interface) = if options.import {
-            (vec![iface.clone()], None)
-        } else {
-            (Vec::new(), Some(iface.clone()))
-        };
-        let dummy = test_helpers::dummy_module(&imports, &[], interface.as_ref());
-        let mut encoder = wit_component::ComponentEncoder::default()
+        let dummy = test_helpers::dummy_module(&interfaces);
+        let wasm = wit_component::ComponentEncoder::default()
             .module(&dummy)?
-            .imports(imports)?;
-        if let Some(iface) = interface {
-            encoder = encoder.interface(iface)?;
-        }
-        let wasm = encoder.encode()?;
+            .interfaces(interfaces.clone(), wit_component::StringEncoding::UTF8)?
+            .encode()?;
         wit_bindgen_core::component::generate(&mut *gen, "input", &wasm, files)
     };
 

crates/wit-component/src/cli.rs

@@ -1,6 +1,9 @@
 //! The WebAssembly component tool command line interface.
 
-use crate::{decode_component_interfaces, ComponentEncoder, InterfacePrinter, StringEncoding};
+use crate::{
+    decode_component_interfaces, ComponentEncoder, ComponentInterfaces, InterfacePrinter,
+    StringEncoding,
+};
 use anyhow::{anyhow, bail, Context, Result};
 use clap::Parser;
 use std::path::{Path, PathBuf};
@@ -119,24 +122,34 @@ impl WitComponentApp {
         let module = wat::parse_file(&self.module)
             .with_context(|| format!("failed to parse module `{}`", self.module.display()))?;
 
+        let mut interfaces = ComponentInterfaces::default();
+        for import in self.imports {
+            let name = import.name.clone();
+            let prev = interfaces.imports.insert(name.clone(), import);
+            if prev.is_some() {
+                bail!("duplicate import interface specified for `{name}`");
+            }
+        }
+        for export in self.exports {
+            let name = export.name.clone();
+            let prev = interfaces.exports.insert(name.clone(), export);
+            if prev.is_some() {
+                bail!("duplicate export interface specified for `{name}`");
+            }
+        }
+        interfaces.default = self.interface;
+
+        let encoding = self.encoding.unwrap_or(StringEncoding::UTF8);
+
         let mut encoder = ComponentEncoder::default()
             .module(&module)?
-            .imports(self.imports)?
-            .exports(self.exports)?
+            .interfaces(interfaces, encoding)?
             .validate(!self.skip_validation);
 
         for (name, wasm) in self.adapters.iter() {
             encoder = encoder.adapter(name, wasm)?;
         }
 
-        if let Some(interface) = self.interface {
-            encoder = encoder.interface(interface)?;
-        }
-
-        if let Some(encoding) = &self.encoding {
-            encoder = encoder.encoding(*encoding);
-        }
-
         let bytes = encoder.encode().with_context(|| {
             format!(
                 "failed to encode a component from module `{}`",

crates/wit-component/src/decoding.rs

@@ -88,7 +88,7 @@ struct InterfaceDecoder<'a> {
 ///
 /// This is more-or-less a "world" and will likely be replaced one day with a
 /// `wit-parser` representation of a world.
-#[derive(Default)]
+#[derive(Clone, Default)]
 pub struct ComponentInterfaces {
     /// The "default export" which is the interface directly exported from the
     /// component at the top level.