various updates to support wasi:http@0.2.0-rc-2023-10-18 (#43)

* various updates to support `wasi:http@0.2.0-rc-2023-10-18`

This fixes various issues:

- Broken generated code indentation for resources in some cases
- Type annotations that refer to non-yet-declared types confuse CPython, so we disable them
  - However, MyPy has no trouble with them, so we enable them by default for the `bindings` subcommand
- Support WIT version annotations (i.e. pass them through to the generated component)
  - This partially addresses #19, but doesn't support importing or exporting multiple versions of the same interface
- Update the `http` example to match `wasi:http@0.2.0-rc-2023-10-18`
- Update to Wasmtime 14 and the latest `wit-parser`, `wit-component`, etc.
  - and update the WASI preview 1 adapter to match

This also bumps the version to 0.6.0.

Note that I've had to remove the `matrix-math` example since `wasmtime-py` does
not yet support resources.  Although the example itself doesn't use them, the
new WASI Preview 1 adapter pulls them in as WASI Preview 2 imports, and there's
no feasible way to work around that.  Ideally, we'd provide the option to allow
users to supply their own adapter, in which case we could use a pre-resource
version of the adapter.  However, that won't work given that pre-initialization
is central to how `componentize-py` works.  Hopefully we can bring back this
example in the future, e.g. when `wasmtime-py` adds support for resources.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

bundle `poll_loop.py` to make it available during pre-init

This module is useful enough that it makes sense to bundle it as part of
`componentize-py`.  Eventually, we may want to distribute it via PyPI as a
helper library, but we'll settle for bundling for now.  It shouldn't add any
overhead for apps that don't `import` it.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

* update `http` example doc comments

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

---------

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Author: dicej

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "componentize-py"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 exclude = ["cpython"]
 
@@ -17,17 +17,15 @@ zstd = "0.11.1"
 componentize-py-shared = { path = "shared" }
 wasmparser = "0.107.0"
 wasm-encoder = "0.29.0"
-# TODO: switch to release once https://github.com/bytecodealliance/wasm-tools/pull/1226 is merged and released:
-wit-parser = { git = "https://github.com/dicej/wasm-tools", branch = "adapter-export-resources" }
-wit-component = { git = "https://github.com/dicej/wasm-tools", branch = "adapter-export-resources" }
+wit-parser = "0.12.2"
+wit-component = "0.17.0"
 indexmap = "2.0.0"
 bincode = "1.3.3"
 heck = "0.4.1"
 pyo3 = { version = "0.18.3", features = ["abi3-py37", "extension-module"], optional = true }
-# TODO: switch to Wasmtime 14 when released:
-wasmtime-wasi = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966" }
-wasi-common = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966" }
-wasmtime = { git = "https://github.com/bytecodealliance/wasmtime", rev = "40c1f9b8b4f962ed763e47943e6ce0a3be8d1966", features = [ "component-model" ] }
+wasmtime-wasi = "14.0.3"
+wasi-common = "14.0.3"
+wasmtime = { version = "14.0.3", features = [ "component-model" ] }
 once_cell = "1.17.1"
 component-init = { git = "https://github.com/dicej/component-init" }
 async-trait = "0.1.68"

Cargo.toml

@@ -0,0 +1,8 @@
+The subdirectory of this directory contains a build of the WASI Preview 1
+component adapter.  It was built from commit `e8766e49` of
+https://github.com/dicej/wasmtime using the
+`ci/build-wasi-preview1-component-adapter.sh` script.
+
+TODO: Switch back to upstream once
+https://github.com/bytecodealliance/wasmtime/pull/7444 has been merged and
+released.

adapters/40c1f9b8/wasi_snapshot_preview1.reactor.wasm

@@ -80,6 +80,14 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> {
             .do_finish()?;
     }
 
+    let path = out_dir.join("bundled.tar.zst");
+
+    if !path.exists() {
+        Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?)
+            .into_inner()?
+            .do_finish()?;
+    }
+
     Ok(())
 }
 
@@ -184,8 +192,24 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> {
     } else {
         bail!("no such directory: {}", path.display())
     }
+
+    let path = repo_dir.join("bundled");
+
+    if path.exists() {
+        let mut builder = Builder::new(Encoder::new(
+            File::create(out_dir.join("bundled.tar.zst"))?,
+            ZSTD_COMPRESSION_LEVEL,
+        )?);
+
+        add(&mut builder, &path, &path)?;
+
+        builder.into_inner()?.do_finish()?;
+    } else {
+        bail!("no such directory: {}", path.display())
+    }
+
     compress(
-        &repo_dir.join("adapters/40c1f9b8"),
+        &repo_dir.join("adapters/e8766e49"),
         "wasi_snapshot_preview1.reactor.wasm",
         out_dir,
         false,

adapters/README.md

@@ -1,4 +1,6 @@
-"""Defines a custom `asyncio` event loop backed by WASI's `poll_oneoff`.
+"""Defines a custom `asyncio` event loop backed by `wasi:io/poll#poll-list`.
+
+This also includes helper classes and functions for working with `wasi:http`.
 
 As of WASI Preview 2, there is not yet a standard for first-class, composable
 asynchronous functions and streams.  We expect that little or none of this
@@ -9,68 +11,104 @@
 import socket
 import subprocess
 
-from proxy.imports import types2 as types, streams2 as streams, poll2 as poll
-from proxy.imports.streams2 import StreamStatus
+from proxy.types import Ok, Err
+from proxy.imports import types, streams, poll, outgoing_handler
+from proxy.imports.types import IncomingBody, OutgoingBody, OutgoingRequest, IncomingResponse
+from proxy.imports.streams import StreamErrorClosed, InputStream
+from proxy.imports.poll import Pollable
 from typing import Optional, cast
 
 # Maximum number of bytes to read at a time
 READ_SIZE: int = 16 * 1024
 
+async def send(request: OutgoingRequest) -> IncomingResponse:
+    """Send the specified request and wait asynchronously for the response."""
+    
+    future = outgoing_handler.handle(request, None)
+
+    while True:
+        response = future.get()
+        if response is None:
+            await register(cast(PollLoop, asyncio.get_event_loop()), future.subscribe())
+        else:
+            if isinstance(response, Ok):
+                if isinstance(response.value, Ok):
+                    return response.value.value
+                else:
+                    raise response.value
+            else:
+                raise response
+
 class Stream:
-    """Reader abstraction over `wasi-cli`'s low-level stream pseudo-resource."""
-    def __init__(self, stream: int):
-        self.pollable = streams.subscribe_to_input_stream(stream)
-        self.stream = stream
-        self.saw_end = False
+    """Reader abstraction over `wasi:http/types#incoming-body`."""
+    def __init__(self, body: IncomingBody):
+        self.body: Optional[IncomingBody] = body
+        self.stream: Optional[InputStream] = body.stream()
 
     async def next(self) -> Optional[bytes]:
         """Wait for the next chunk of data to arrive on the stream.
 
         This will return `None` when the end of the stream has been reached.
         """
-        if self.saw_end:
-            return None
-        else:
-            while True:
-                buffer, status = streams.read(self.stream, READ_SIZE)
-                if status == StreamStatus.ENDED:
-                    types.finish_incoming_stream(self.stream)
-                    self.saw_end = True
-
-                if buffer:
-                    return buffer
-                elif status == StreamStatus.ENDED:
+        while True:
+            try:
+                if self.stream is None:
                     return None
                 else:
-                    await register(cast(PollLoop, asyncio.get_event_loop()), self.pollable)
+                    buffer = self.stream.read(READ_SIZE)
+                    if len(buffer) == 0:
+                        await register(cast(PollLoop, asyncio.get_event_loop()), self.stream.subscribe())
+                    else:
+                        return buffer
+            except Err as e:
+                if isinstance(e.value, StreamErrorClosed):
+                    if self.stream is not None:
+                        self.stream.drop()
+                        self.stream = None
+                    if self.body is not None:
+                        IncomingBody.finish(self.body)
+                        self.body = None
+                else:
+                    raise e
 
 class Sink:
-    """Writer abstraction over `wasi-cli`'s low-level stream pseudo-resource."""
-    def __init__(self, stream: int):
-        self.pollable = streams.subscribe_to_output_stream(stream)
-        self.stream = stream
+    """Writer abstraction over `wasi-http/types#outgoing-body`."""
+    def __init__(self, body: OutgoingBody):
+        self.body = body
+        self.stream = body.write()
 
     async def send(self, chunk: bytes):
-        """Write the specified bytes to the stream.
+        """Write the specified bytes to the sink.
 
-        This may need to yield according to the backpressure requirements of the stream.
+        This may need to yield according to the backpressure requirements of the sink.
         """
         offset = 0
+        flushing = False
         while True:
-            count = streams.write(self.stream, chunk[offset:])
-            offset += count
-            if offset == len(chunk):
-                return
+            count = self.stream.check_write()
+            if count == 0:
+                await register(cast(PollLoop, asyncio.get_event_loop()), self.stream.subscribe())
+            elif offset == len(chunk):
+                if flushing:
+                    return
+                else:
+                    self.stream.flush()
+                    flushing = True
             else:
-                await register(cast(PollLoop, asyncio.get_event_loop()), self.pollable)
+                count = min(count, len(chunk) - offset)
+                self.stream.write(chunk[offset:offset+count])
+                offset += count
 
     def close(self):
         """Close the stream, indicating no further data will be written."""
-        
-        types.finish_outgoing_stream(self.stream)
+
+        self.stream.drop()
+        self.stream = None
+        OutgoingBody.finish(self.body, None)
+        self.body = None
 
 class PollLoop(asyncio.AbstractEventLoop):
-    """Custom `asyncio` event loop backed by WASI's `poll_oneoff` function."""
+    """Custom `asyncio` event loop backed by `wasi:io/poll#poll-list`."""
 
     def __init__(self):
         self.wakers = []
@@ -96,8 +134,13 @@ def run_until_complete(self, future):
                 [pollables, wakers] = list(map(list, zip(*self.wakers)))
 
                 new_wakers = []
-                for (ready, pollable), waker in zip(zip(poll.poll_oneoff(pollables), pollables), wakers):
+                ready = [False] * len(pollables)
+                for index in poll.poll_list(pollables):
+                    ready[index] = True
+                
+                for (ready, pollable), waker in zip(zip(ready, pollables), wakers):
                     if ready:
+                        pollable.drop()
                         waker.set_result(None)
                     else:
                         new_wakers.append((pollable, waker))
@@ -319,7 +362,7 @@ def default_exception_handler(self, context):
     def set_debug(self, enabled):
         raise NotImplementedError
 
-async def register(loop: PollLoop, pollable: int):
+async def register(loop: PollLoop, pollable: Pollable):
     waker = loop.create_future()
     loop.wakers.append((pollable, waker))
     await waker

adapters/e8766e49/wasi_snapshot_preview1.reactor.wasm

@@ -1,24 +1,27 @@
 # Example: `http`
 
-This is an example of how to use [componentize-py] and [Spin] to build and run a
-Python-based component targetting the [wasi-http] `proxy` world.
+This is an example of how to use [componentize-py] and [Wasmtime] to build and
+run a Python-based component targetting the [wasi-http] `proxy` world.
 
 Note that, as of this writing, neither `wasi-http` nor the portions of
 `wasi-cli` on which it is based have stabilized.  Here we use a snapshot of both,
 which may differ from later revisions.
 
 [componentize-py]: https://github.com/bytecodealliance/componentize-py
-[Spin]: https://github.com/fermyon/spin
+[Wasmtime]: https://github.com/bytecodealliance/wasmtime
 [wasi-http]: https://github.com/WebAssembly/wasi-http
 
 ## Prerequisites
 
-* `dicej/spin` branch `wasi-http-wasmtime-2ad057d7`
-* `componentize-py` 0.5.0
-* `Rust`, for installing `Spin`
+* `Wasmtime` 14.0.3 (later versions may use a different, incompatible `wasi-http` snapshot)
+* `componentize-py` 0.6.0
+
+Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`.  If
+you don't have `cargo`, you can download and install from
+https://github.com/bytecodealliance/wasmtime/releases/tag/v14.0.3.
 
 ```
-cargo install --locked --git https://github.com/dicej/spin --branch wasi-http-wasmtime-2ad057d7 spin-cli
+cargo install --version 14.0.3 wasmtime-cli
 pip install componentize-py
 ```
 
@@ -27,13 +30,14 @@ pip install componentize-py
 First, build the app and run it:
 
 ```
-spin build --up
+componentize-py -d wit -w proxy componentize app -o http.wasm
+wasmtime serve http.wasm
 ```
 
 Then, in another terminal, use cURL to send a request to the app:
 
 ```
-curl -i -H 'content-type: text/plain' --data-binary @- http://127.0.0.1:3000/echo <<EOF
+curl -i -H 'content-type: text/plain' --data-binary @- http://127.0.0.1:8080/echo <<EOF
 ’Twas brillig, and the slithy toves
       Did gyre and gimble in the wabe:
 All mimsy were the borogoves,
@@ -52,7 +56,7 @@ curl -i \
     -H 'url: https://webassembly.github.io/spec/core/' \
     -H 'url: https://www.w3.org/groups/wg/wasm/' \
     -H 'url: https://bytecodealliance.org/' \
-    http://127.0.0.1:3000/hash-all
+    http://127.0.0.1:8080/hash-all
 ```
 
 If you run into any problems, please file an issue!