add test for multiworld scenario (#77)

* add test for multiworld scenario

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

* bump version to 0.12.0

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

---------

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Author: dicej

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.12.0-rc1"
+version = "0.12.0"
 edition = "2021"
 exclude = ["cpython"]
 

Cargo.toml

@@ -9,23 +9,23 @@ run a Python-based component targetting the [wasi-cli] `command` world.
 
 ## Prerequisites
 
-* `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.12.0rc1
+* `Wasmtime` 18.0.0 or later
+* `componentize-py` 0.12.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
-https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
+https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
-cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.12.0rc1
+cargo install --version 18.0.0 wasmtime-cli
+pip install componentize-py==0.12.0
 ```
 
 ## Running the demo
 
 ```
 componentize-py -d ../../wit -w wasi:cli/command@0.2.0 componentize app -o cli.wasm
-wasmtime run --wasm component-model cli.wasm
+wasmtime run cli.wasm
 ```
 
 The `wasmtime run` command above should print "Hello, world!".

examples/cli/README.md

@@ -9,16 +9,16 @@ run a Python-based component targetting the [wasi-http] `proxy` world.
 
 ## Prerequisites
 
-* `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.12.0rc1
+* `Wasmtime` 18.0.0 or later
+* `componentize-py` 0.12.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
-https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
+https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
-cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.12.0rc1
+cargo install --version 18.0.0 wasmtime-cli
+pip install componentize-py==0.12.0
 ```
 
 ## Running the demo

examples/http/README.md

@@ -10,20 +10,20 @@ within a guest component.
 
 ## Prerequisites
 
-* `wasmtime` 17.0.0 or later
-* `componentize-py` 0.12.0rc1
+* `wasmtime` 18.0.0 or later
+* `componentize-py` 0.12.0
 * `NumPy`, built for WASI
 
 Note that we use an unofficial build of NumPy since the upstream project does
 not yet publish WASI builds.
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
-https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
+https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
-cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.12.0rc1
+cargo install --version 18.0.0 wasmtime-cli
+pip install componentize-py==0.12.0
 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.1/numpy-wasi.tar.gz
 tar xf numpy-wasi.tar.gz
 ```
@@ -32,7 +32,7 @@ tar xf numpy-wasi.tar.gz
 
 ```
 componentize-py -d ../../wit -w matrix-math componentize app -o matrix-math.wasm
-wasmtime run --wasm component-model matrix-math.wasm '[[1, 2], [4, 5], [6, 7]]' '[[1, 2, 3], [4, 5, 6]]'
+wasmtime run matrix-math.wasm '[[1, 2], [4, 5], [6, 7]]' '[[1, 2, 3], [4, 5, 6]]'
 ```
 
 The second command above should print the following:

examples/matrix-math/README.md

@@ -10,16 +10,16 @@ making an outbound TCP request using `wasi-sockets`.
 
 ## Prerequisites
 
-* `Wasmtime` 17.0.0 or later
-* `componentize-py` 0.12.0rc1
+* `Wasmtime` 18.0.0 or later
+* `componentize-py` 0.12.0
 
 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
 you don't have `cargo`, you can download and install from
-https://github.com/bytecodealliance/wasmtime/releases/tag/v17.0.0.
+https://github.com/bytecodealliance/wasmtime/releases/tag/v18.0.0.
 
 ```
-cargo install --version 17.0.0 wasmtime-cli
-pip install componentize-py==0.12.0rc1
+cargo install --version 18.0.0 wasmtime-cli
+pip install componentize-py==0.12.0
 ```
 
 ## Running the demo
@@ -35,7 +35,7 @@ Now, build and run the example, using the same port you gave to `netcat`.
 
 ```
 componentize-py -d ../../wit -w wasi:cli/command@0.2.0 componentize app -o tcp.wasm
-wasmtime run --wasm component-model --wasi inherit-network tcp.wasm 127.0.0.1:3456
+wasmtime run --wasi inherit-network tcp.wasm 127.0.0.1:3456
 ```
 
 The program will open a TCP connection, send a message, and wait to receive a

examples/tcp/README.md

@@ -7,7 +7,7 @@ features = ["pyo3/extension-module"]
 
 [project]
 name = "componentize-py"
-version = "0.12.0rc1"
+version = "0.12.0"
 description = "Tool to package Python applications as WebAssembly components"
 readme = "README.md"
 license = { file = "LICENSE" }

pyproject.toml

@@ -214,6 +214,7 @@ pub async fn componentize(
     output_path: &Path,
     add_to_linker: Option<&dyn Fn(&mut Linker<Ctx>) -> Result<()>>,
 ) -> Result<()> {
+    // Untar the embedded copy of the Python standard library into a temporary directory
     let stdlib = tempfile::tempdir()?;
 
     Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!(
@@ -222,6 +223,7 @@ pub async fn componentize(
     ))))?)
     .unpack(stdlib.path())?;
 
+    // Untar the embedded copy of helper utilties into a temporary directory
     let bundled = tempfile::tempdir()?;
 
     Archive::new(Decoder::new(Cursor::new(include_bytes!(concat!(
@@ -230,6 +232,9 @@ pub async fn componentize(
     ))))?)
     .unpack(bundled.path())?;
 
+    // Search `python_path` for native extension libraries and/or componentize-py.toml files.  Packages containing
+    // the latter may contain their own WIT files defining their own worlds (in addition to what the caller
+    // specified as paramters), which we'll try to match up with `module_worlds` in the next step.
     let mut raw_configs = Vec::new();
     let mut library_path = Vec::with_capacity(python_path.len());
     for path in python_path {
@@ -244,6 +249,12 @@ pub async fn componentize(
         library_path.push((*path, libraries));
     }
 
+    // Validate the paths parsed from any componentize-py.toml files discovered above and match them up with
+    // `module_worlds` entries.  Note that we use an `IndexMap` to preserve the order specified in `module_worlds`,
+    // which is required to be topologically sorted with respect to package dependencies.
+    //
+    // For any packages which contain componentize-py.toml files but no corresponding `module_worlds` entry, we use
+    // the `world` parameter as a default.
     let configs = {
         let mut configs = raw_configs
             .into_iter()
@@ -279,6 +290,9 @@ pub async fn componentize(
         ordered
     };
 
+    // Next, iterate over all the WIT directories, merging them into a single `Resolve`, and matching Python
+    // packages to `WorldId`s.
+
     let (mut resolve, mut main_world) = if let Some(path) = wit_path {
         let (resolve, world) = parse_wit(path, world)?;
         (Some(resolve), Some(world))
@@ -313,11 +327,19 @@ pub async fn componentize(
     let resolve = if let Some(resolve) = resolve {
         resolve
     } else {
-        let (my_resolve, world) = parse_wit(Path::new("wit"), world)?;
+        // If no WIT directory was provided as a parameter and none were referenced by Python packages, use ./wit
+        // by default.
+        let (my_resolve, world) = parse_wit(Path::new("wit"), world).context(
+            "no WIT files found; please specify the directory or file \
+             containing the WIT world you wish to target",
+        )?;
         main_world = Some(world);
         my_resolve
     };
 
+    // Extract relevant metadata from the `Resolve` into a `Summary` instance, which we'll use to generate Wasm-
+    // and Python-level bindings.
+
     let worlds = configs
         .values()
         .filter_map(|(_, world)| *world)
@@ -326,6 +348,7 @@ pub async fn componentize(
 
     let summary = Summary::try_new(&resolve, &worlds)?;
 
+    // Link all the libraries (including any native extensions) into a single component.
     let mut linker = wit_component::Linker::default()
         .validate(true)
         .library(
@@ -427,6 +450,10 @@ pub async fn componentize(
 
     let component = linker.encode()?;
 
+    // Pre-initialize the component by running it through `component_init::initialize`.  Currently, this is the
+    // application's first and only chance to load any standard or third-party modules since we do not yet include
+    // a virtual filesystem in the component to make those modules available at runtime.
+
     let stdout = MemoryOutputPipe::new(10000);
     let stderr = MemoryOutputPipe::new(10000);
 
@@ -452,6 +479,7 @@ pub async fn componentize(
             "bundled",
         );
 
+    // Generate guest mounts for each host directory in `python_path`.
     for (index, path) in python_path.iter().enumerate() {
         wasi.preopened_dir(
             Dir::open_ambient_dir(path, cap_std::ambient_authority())
@@ -462,6 +490,9 @@ pub async fn componentize(
         );
     }
 
+    // For each Python package with a `componentize-py.toml` file that specifies where generated bindings for that
+    // package should be placed, generate the bindings and place them as indicated.
+
     let mut world_dir_mounts = Vec::new();
     let mut locations = Locations::default();
     let mut saw_main_world = false;
@@ -518,6 +549,7 @@ pub async fn componentize(
         ));
     }
 
+    // If the caller specified a world and we haven't already generated bindings for it above, do so now.
     if let (Some(world), false) = (main_world, saw_main_world) {
         let module = resolve.worlds[world].name.to_snake_case();
         let world_dir = tempfile::tempdir()?;
@@ -539,8 +571,12 @@ pub async fn componentize(
         }
     }
 
+    // Generate a `Symbols` object containing metadata to be passed to the pre-init function.  The runtime library
+    // will use this to look up types and functions that will later be referenced by the generated Wasm code.
     let symbols = summary.collect_symbols(&locations);
 
+    // Finally, pre-initialize the component writing the result to `output_path`.
+
     let python_path = (0..python_path.len())
         .map(|index| format!("/{index}"))
         .collect::<Vec<_>>()

src/lib.rs

@@ -1463,21 +1463,10 @@ class {camel}(Protocol):
                 };
 
                 let aliases = if let (Some(code), false) = (code.as_ref(), names.is_empty()) {
-                    let aliases = iter::once(format!(
-                        "import {}",
-                        if let Some((start, _)) = world_module.split_once('.') {
-                            start
-                        } else {
-                            world_module
-                        }
-                    ))
-                    .chain(
-                        names
-                            .iter()
-                            .map(|name| format!("{name} = {world_module}.{name}")),
-                    )
-                    .collect::<Vec<_>>()
-                    .join("\n");
+                    let aliases = iter::once(world_module_import(world_module, "peer"))
+                        .chain(names.iter().map(|name| format!("{name} = peer.{name}")))
+                        .collect::<Vec<_>>()
+                        .join("\n");
 
                     Some(match code {
                         Code::Shared(_) => Code::Shared(aliases),
@@ -1711,21 +1700,13 @@ import weakref
         {
             let mut file = File::create(path.join("types.py"))?;
             if let Some(module) = locations.types_module.as_ref() {
-                writeln!(
-                    file,
-                    "import {}",
-                    if let Some((start, _)) = module.split_once('.') {
-                        start
-                    } else {
-                        module
-                    }
-                )?;
+                writeln!(file, "{}", world_module_import(module, "peer"))?;
                 write!(
                     file,
-                    "Some = {module}.Some
-Ok = {module}.types.Ok
-Err = {module}.types.Err
-Result = {module}.types.Result
+                    "Some = peer.types.Some
+Ok = peer.types.Ok
+Err = peer.types.Err
+Result = peer.types.Result
 "
                 )?;
             } else {
@@ -2162,3 +2143,11 @@ fn matches_resource(function: &MyFunction, resource: TypeId, direction: Directio
         _ => false,
     }
 }
+
+fn world_module_import(name: &str, alias: &str) -> String {
+    if let Some((front, rear)) = name.rsplit_once('.') {
+        format!("from {front} import {rear} as {alias}")
+    } else {
+        format!("import {name} as {alias}")
+    }
+}