add support for targeting multiple worlds with a single component (#76)

A while back, I added support for `componentize-py.toml` files in Python
packages, which allows developers to embed WIT files in their Python packages
and also specify where the bindings should be generated with respect to the
Python module hierarchy.  This is useful when publishing SDK packages for a WIT
world, which may include additional, handwritten code on top of the generated
bindings.  However, it wasn't possible to use more than one such package at a
time, nor was it possible to use such a package and _also_ target an additional,
application-level WIT world beyond those defined in the package.

This commit removes those instructions, allowing the user to specify multiple
worlds (up to one per package, plus one additional, application-level world)
when generating a component.  Since those worlds might overlap, we merge them
together using `wit_parser::Resolve::merge` and then generate bindings for each
world such that the bindings for a given world may alias bindings generated for
an earlier world.

This required quite a bit of refactoring, and the code responsible for juggling
all the WIT directories, worlds, and bindings is a bit awkward since it's trying
to handle a wide range of use cases.  It also needs more (automated) testing.
In short, it's not fully baked yet, but I want to get a pre-release out so folks
can try it while I clean things up.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Author: dicej

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.11.3"
+version = "0.12.0-rc1"
 edition = "2021"
 exclude = ["cpython"]
 

Cargo.toml

@@ -10,15 +10,15 @@ run a Python-based component targetting the [wasi-cli] `command` world.
 ## Prerequisites
 
 * `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.11.3
+* `componentize-py` 0.12.0rc1
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
 
 ```
 cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.11.3
+pip install componentize-py==0.12.0rc1
 ```
 
 ## Running the demo

examples/cli/README.md

@@ -10,15 +10,15 @@ run a Python-based component targetting the [wasi-http] `proxy` world.
 ## Prerequisites
 
 * `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.11.3
+* `componentize-py` 0.12.0rc1
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
 
 ```
 cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.11.3
+pip install componentize-py==0.12.0rc1
 ```
 
 ## Running the demo

examples/http/README.md

@@ -11,7 +11,7 @@ within a guest component.
 ## Prerequisites
 
 * `wasmtime` 17.0.0 or later
-* `componentize-py` 0.11.3
+* `componentize-py` 0.12.0rc1
 * `NumPy`, built for WASI
 
 Note that we use an unofficial build of NumPy since the upstream project does
@@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
 
 ```
 cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.11.3
+pip install componentize-py==0.12.0rc1
 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz
 tar xf numpy-wasi.tar.gz
 ```

examples/matrix-math/README.md

@@ -11,15 +11,15 @@ making an outbound TCP request using `wasi-sockets`.
 ## Prerequisites
 
 * `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.11.3
+* `componentize-py` 0.12.0rc1
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
 https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
 
 ```
 cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.11.3
+pip install componentize-py==0.12.0rc1
 ```
 
 ## Running the demo

examples/tcp/README.md

@@ -7,7 +7,7 @@ features = ["pyo3/extension-module"]
 
 [project]
 name = "componentize-py"
-version = "0.11.3"
+version = "0.12.0rc1"
 description = "Tool to package Python applications as WebAssembly components"
 readme = "README.md"
 license = { file = "LICENSE" }

pyproject.toml

@@ -28,7 +28,11 @@ struct MemInfo {
     table_alignment: u32,
 }
 
-pub fn make_bindings(resolve: &Resolve, world: WorldId, summary: &Summary) -> Result<Vec<u8>> {
+pub fn make_bindings(
+    resolve: &Resolve,
+    worlds: &IndexSet<WorldId>,
+    summary: &Summary,
+) -> Result<Vec<u8>> {
     // TODO: deduplicate types
     let mut types = TypeSection::new();
     let mut imports = ImportSection::new();
@@ -363,21 +367,21 @@ pub fn make_bindings(resolve: &Resolve, world: WorldId, summary: &Summary) -> Re
         name: Cow::Borrowed("name"),
         data: Cow::Borrowed(&names_data),
     });
-    result.section(&CustomSection {
-        name: Cow::Owned(format!("component-type:{}", resolve.worlds[world].name)),
-        data: Cow::Owned(metadata::encode(
-            resolve,
-            world,
-            wit_component::StringEncoding::UTF8,
-            None,
-        )?),
-    });
+    for &world in worlds {
+        result.section(&CustomSection {
+            name: Cow::Owned(format!("component-type:{}", resolve.worlds[world].name)),
+            data: Cow::Owned(metadata::encode(
+                resolve,
+                world,
+                wit_component::StringEncoding::UTF8,
+                None,
+            )?),
+        });
+    }
 
     let result = result.finish();
 
     wasmparser::validate(&result)?;
 
-    std::fs::write("/tmp/bindings.so", &result)?;
-
     Ok(result)
 }

src/bindings.rs

@@ -61,6 +61,17 @@ pub struct Componentize {
     #[arg(short = 'p', long, default_value = ".")]
     pub python_path: Vec<String>,
 
+    /// Specify which world to use with which Python module.  May be specified more than once.
+    ///
+    /// Some Python modules (e.g. SDK wrappers around WIT APIs) may contain `componentize-py.toml` files which
+    /// point to embedded WIT files, and those may define multiple WIT worlds.  In this case, it may be necessary
+    /// to specify which world on a module-by-module basis.
+    ///
+    /// Note that these must be specified in topological order (i.e. if a module containing WIT files depends on
+    /// other modules containing WIT files, it must be listed after all its dependencies).
+    #[arg(short = 'm', long, value_parser = parse_module_world)]
+    pub module_worlds: Vec<(String, String)>,
+
     /// Output file to which to write the resulting component
     #[arg(short = 'o', long, default_value = "index.wasm")]
     pub output: PathBuf,
@@ -80,6 +91,13 @@ pub struct Bindings {
     pub world_module: Option<String>,
 }
 
+fn parse_module_world(s: &str) -> Result<(String, String), String> {
+    let (k, v) = s
+        .split_once('=')
+        .ok_or_else(|| format!("expected string of form `<key>=<value>`; got `{s}`"))?;
+    Ok((k.to_string(), v.to_string()))
+}
+
 pub fn run<T: Into<OsString> + Clone, I: IntoIterator<Item = T>>(args: I) -> Result<()> {
     let options = Options::parse_from(args);
     match options.command {
@@ -115,6 +133,11 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> {
         common.wit_path.as_deref(),
         common.world.as_deref(),
         &python_path.iter().map(|s| s.as_str()).collect::<Vec<_>>(),
+        &componentize
+            .module_worlds
+            .iter()
+            .map(|(k, v)| (k.as_str(), v.as_str()))
+            .collect::<Vec<_>>(),
         &componentize.app_name,
         &componentize.output,
         None,