Skip to content

Commit 6c7ee21

Browse files
feat(client): Adds support for attaching annotations to manifests for OCI
This maps the metadata within a component (if it is present) to the appropriate annotations on the manifest. This also sets up some basic testing helpers for wkg e2e tests Signed-off-by: Taylor Thomas <taylor@cosmonic.com>
1 parent 0c141a6 commit 6c7ee21

File tree

13 files changed

+967
-4
lines changed

13 files changed

+967
-4
lines changed

.github/workflows/ci.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ jobs:
2424
- name: Run client tests
2525
working-directory: ./crates/wasm-pkg-client
2626
run: cargo test ${{ matrix.additional_test_flags }}
27+
- name: Run wkg tests
28+
working-directory: ./crates/wkg
29+
run: cargo test ${{ matrix.additional_test_flags }}
2730
- name: Run other tests
28-
run: cargo test --workspace --exclude wasm-pkg-client
31+
run: cargo test --workspace --exclude wasm-pkg-client --exclude wkg
2932
- name: Run cargo clippy
3033
run: cargo clippy --all --workspace

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ serde = { version = "1.0", features = ["derive"] }
2626
serde_json = "1"
2727
sha2 = "0.10"
2828
tempfile = "3.10.1"
29+
testcontainers = { version = "0.23" }
2930
thiserror = "1.0"
3031
tokio = "1.35.1"
3132
tokio-process = "0.2.5"

crates/wasm-pkg-client/Cargo.toml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,11 @@ tracing-subscriber = { workspace = true }
3636
url = "2.5.0"
3737
warg-client = "0.9.0"
3838
warg-crypto = "0.9.0"
39+
wasm-metadata = { workspace = true }
3940
warg-protocol = "0.9.0"
4041
wasm-pkg-common = { workspace = true, features = ["metadata-client", "tokio"] }
4142
wit-component = { workspace = true }
4243

4344
[dev-dependencies]
44-
tempfile = "3"
45-
testcontainers = "0.23"
45+
tempfile = { workspace = true }
46+
testcontainers = { workspace = true }

crates/wasm-pkg-client/src/oci/publisher.rs

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
use std::collections::BTreeMap;
2+
13
use oci_client::{Reference, RegistryOperation};
24
use tokio::io::AsyncReadExt;
5+
use wasm_metadata::LinkType;
36

47
use crate::publisher::PackagePublisher;
58
use crate::{PackageRef, PublishingSource, Version};
@@ -19,13 +22,50 @@ impl PackagePublisher for OciBackend {
1922
// to remove this and use the stream directly.
2023
let mut buf = Vec::new();
2124
data.read_to_end(&mut buf).await?;
25+
let meta = wasm_metadata::RegistryMetadata::from_wasm(&buf).map_err(|e| {
26+
crate::Error::InvalidComponent(anyhow::anyhow!("Unable to parse component: {e}"))
27+
})?;
2228
let (config, layer) = oci_wasm::WasmConfig::from_raw_component(buf, None)
2329
.map_err(crate::Error::InvalidComponent)?;
30+
let mut annotations = BTreeMap::from_iter([(
31+
"org.opencontainers.image.version".to_string(),
32+
version.to_string(),
33+
)]);
34+
if let Some(meta) = meta {
35+
if let Some(desc) = meta.get_description() {
36+
annotations.insert(
37+
"org.opencontainers.image.description".to_string(),
38+
desc.to_owned(),
39+
);
40+
}
41+
if let Some(licenses) = meta.get_license() {
42+
annotations.insert(
43+
"org.opencontainers.image.licenses".to_string(),
44+
licenses.to_owned(),
45+
);
46+
}
47+
if let Some(sources) = meta.get_links() {
48+
for link in sources {
49+
if link.ty == LinkType::Repository {
50+
annotations.insert(
51+
"org.opencontainers.image.source".to_string(),
52+
link.value.to_owned(),
53+
);
54+
}
55+
if link.ty == LinkType::Homepage {
56+
annotations.insert(
57+
"org.opencontainers.image.url".to_string(),
58+
link.value.to_owned(),
59+
);
60+
}
61+
}
62+
}
63+
}
2464

2565
let reference: Reference = self.make_reference(package, Some(version));
2666
let auth = self.auth(&reference, RegistryOperation::Push).await?;
2767
self.client
28-
.push(&reference, &auth, layer, config, None)
68+
.push(&reference, &auth, layer, config, Some(annotations))
2969
.await
3070
.map_err(crate::Error::RegistryError)?;
3171
Ok(())

crates/wkg/Cargo.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ authors.workspace = true
88
license.workspace = true
99
readme = "../../README.md"
1010

11+
[features]
12+
default = ["_local"]
13+
# An internal feature for making sure e2e tests can run locally but not in CI for Mac or Windows
14+
_local = []
15+
1116
[dependencies]
1217
anyhow = { workspace = true }
1318
clap = { version = "4.5", features = ["derive", "wrap_help", "env"] }
@@ -29,3 +34,5 @@ wasm-pkg-core = { workspace = true }
2934
[dev-dependencies]
3035
base64 = { workspace = true }
3136
serde_json = { workspace = true }
37+
tempfile = { workspace = true }
38+
testcontainers = { workspace = true }

crates/wkg/tests/common.rs

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
use std::{
2+
collections::HashMap,
3+
net::{Ipv4Addr, SocketAddrV4},
4+
path::{Path, PathBuf},
5+
};
6+
7+
use oci_client::client::ClientConfig;
8+
use testcontainers::{
9+
core::{IntoContainerPort, WaitFor},
10+
runners::AsyncRunner,
11+
ContainerAsync, GenericImage, ImageExt,
12+
};
13+
use tokio::{net::TcpListener, process::Command};
14+
use wasm_pkg_client::{oci::OciRegistryConfig, Config, CustomConfig, Registry, RegistryMetadata};
15+
16+
/// Returns an open port on localhost
17+
pub async fn find_open_port() -> u16 {
18+
TcpListener::bind(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))
19+
.await
20+
.expect("failed to bind random port")
21+
.local_addr()
22+
.map(|addr| addr.port())
23+
.expect("failed to get local address from opened TCP socket")
24+
}
25+
26+
/// Starts a registry container on an open port, returning a [`Config`] with registry auth
27+
/// configured for it and the name of the registry. The container handle is also returned as it must
28+
/// be kept in scope
29+
pub async fn start_registry() -> (Config, Registry, ContainerAsync<GenericImage>) {
30+
let port = find_open_port().await;
31+
let container = GenericImage::new("registry", "2")
32+
.with_wait_for(WaitFor::message_on_stderr("listening on [::]:5000"))
33+
.with_mapped_port(port, 5000.tcp())
34+
.start()
35+
.await
36+
.expect("Failed to start test container");
37+
38+
let registry: Registry = format!("localhost:{}", port).parse().unwrap();
39+
let mut config = Config::empty();
40+
// Make sure we add wasi. The default fallbacks don't get written to disk, which we use in
41+
// tests, so we just start with an empty config and write this into it
42+
config.set_namespace_registry(
43+
"wasi".parse().unwrap(),
44+
wasm_pkg_client::RegistryMapping::Registry("wasi.dev".parse().unwrap()),
45+
);
46+
let reg_conf = config.get_or_insert_registry_config_mut(&registry);
47+
reg_conf
48+
.set_backend_config(
49+
"oci",
50+
OciRegistryConfig {
51+
client_config: ClientConfig {
52+
protocol: oci_client::client::ClientProtocol::Http,
53+
..Default::default()
54+
},
55+
credentials: None,
56+
},
57+
)
58+
.unwrap();
59+
60+
(config, registry, container)
61+
}
62+
63+
/// Clones the given config, mapping the namespace to the given registry at the top level
64+
pub fn map_namespace(config: &Config, namespace: &str, registry: &Registry) -> Config {
65+
let mut config = config.clone();
66+
let mut metadata = RegistryMetadata::default();
67+
metadata.preferred_protocol = Some("oci".to_string());
68+
let mut meta = serde_json::Map::new();
69+
meta.insert("registry".to_string(), registry.to_string().into());
70+
metadata.protocol_configs = HashMap::from_iter([("oci".to_string(), meta)]);
71+
config.set_namespace_registry(
72+
namespace.parse().unwrap(),
73+
wasm_pkg_client::RegistryMapping::Custom(CustomConfig {
74+
registry: registry.to_owned(),
75+
metadata,
76+
}),
77+
);
78+
config
79+
}
80+
81+
/// A loaded fixture with helpers for running wkg tests
82+
pub struct Fixture {
83+
pub temp_dir: tempfile::TempDir,
84+
pub fixture_path: PathBuf,
85+
}
86+
87+
impl Fixture {
88+
/// Returns a base `wkg` command for running tests with the current directory set to the loaded
89+
/// fixture and with a separate wkg cache dir
90+
pub fn command(&self) -> Command {
91+
let mut cmd = Command::new(env!("CARGO_BIN_EXE_wkg"));
92+
cmd.current_dir(&self.fixture_path);
93+
cmd.env("WKG_CACHE_DIR", self.temp_dir.path().join("cache"));
94+
cmd
95+
}
96+
97+
/// Same as [`Fixture::command`] but also writes the given config to disk and sets the
98+
/// `WKG_CONFIG` environment variable to the path of the config
99+
pub async fn command_with_config(&self, config: &Config) -> Command {
100+
let config_path = self.temp_dir.path().join("config.toml");
101+
config
102+
.to_file(&config_path)
103+
.await
104+
.expect("failed to write config");
105+
let mut cmd = self.command();
106+
cmd.env("WKG_CONFIG_FILE", config_path);
107+
cmd
108+
}
109+
}
110+
111+
/// Gets the path to the fixture
112+
pub fn fixture_dir() -> PathBuf {
113+
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
114+
.join("tests")
115+
.join("fixtures")
116+
}
117+
118+
/// Loads the fixture with the given name into a temporary directory. This will copy the fixture
119+
/// from the tests/fixtures directory into a temporary directory and return the tempdir containing
120+
/// that directory (and its path)
121+
pub async fn load_fixture(fixture: &str) -> Fixture {
122+
let temp_dir = tempfile::tempdir().expect("Failed to create tempdir");
123+
let fixture_path = fixture_dir().join(fixture);
124+
// This will error if it doesn't exist, which is what we want
125+
tokio::fs::metadata(&fixture_path)
126+
.await
127+
.expect("Fixture does not exist or couldn't be read");
128+
let copied_path = temp_dir.path().join(fixture_path.file_name().unwrap());
129+
copy_dir(&fixture_path, &copied_path)
130+
.await
131+
.expect("Failed to copy fixture");
132+
Fixture {
133+
temp_dir,
134+
fixture_path: copied_path,
135+
}
136+
}
137+
138+
#[allow(dead_code)]
139+
async fn copy_dir(source: impl AsRef<Path>, destination: impl AsRef<Path>) -> anyhow::Result<()> {
140+
tokio::fs::create_dir_all(&destination).await?;
141+
let mut entries = tokio::fs::read_dir(source).await?;
142+
while let Some(entry) = entries.next_entry().await? {
143+
let filetype = entry.file_type().await?;
144+
if filetype.is_dir() {
145+
// Skip the deps directory in case it is there from debugging
146+
if entry.path().file_name().unwrap_or_default() == "deps" {
147+
continue;
148+
}
149+
Box::pin(copy_dir(
150+
entry.path(),
151+
destination.as_ref().join(entry.file_name()),
152+
))
153+
.await?;
154+
} else {
155+
let path = entry.path();
156+
let extension = path.extension().unwrap_or_default();
157+
// Skip any .lock files that might be there from debugging
158+
if extension == "lock" {
159+
continue;
160+
}
161+
tokio::fs::copy(path, destination.as_ref().join(entry.file_name())).await?;
162+
}
163+
}
164+
Ok(())
165+
}

crates/wkg/tests/e2e.rs

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
mod common;
2+
3+
#[cfg(any(target_os = "linux", feature = "_local"))]
4+
// NOTE: These are only run on linux for CI purposes, because they rely on the docker client being
5+
// available, and for various reasons this has proven to be problematic on both the Windows and
6+
// MacOS runners due to it not being installed (yay licensing).
7+
#[tokio::test]
8+
async fn build_and_publish_with_metadata() {
9+
use oci_client::{client::ClientConfig, manifest::OciManifest, Reference};
10+
11+
let (config, registry, _container) = common::start_registry().await;
12+
13+
let fixture = common::load_fixture("wasi-http").await;
14+
15+
let status = fixture
16+
.command_with_config(&config)
17+
.await
18+
.args(["wit", "build"])
19+
.status()
20+
.await
21+
.expect("Should be able to build wit packagee");
22+
assert!(status.success(), "Build should succeed");
23+
24+
let namespace_mapped_config = common::map_namespace(&config, "wasi", &registry);
25+
let status = fixture
26+
.command_with_config(&namespace_mapped_config)
27+
.await
28+
.args(["publish", "wasi:http@0.2.0.wasm"])
29+
.status()
30+
.await
31+
.expect("Should be able to publish wit package");
32+
assert!(status.success(), "Publish should succeed");
33+
34+
// Now fetch the manifest and verify the annotations are present
35+
let client = oci_client::Client::new(ClientConfig {
36+
protocol: oci_client::client::ClientProtocol::Http,
37+
..Default::default()
38+
});
39+
40+
let reference: Reference = format!("{registry}/wasi/http:0.2.0").parse().unwrap();
41+
let (manifest, _) = client
42+
.pull_manifest(&reference, &oci_client::secrets::RegistryAuth::Anonymous)
43+
.await
44+
.expect("Should be able to fetch manifest");
45+
46+
let manifest = if let OciManifest::Image(m) = manifest {
47+
m
48+
} else {
49+
panic!("Manifest should be an image manifest");
50+
};
51+
52+
let annotations = manifest
53+
.annotations
54+
.expect("Manifest should have annotations");
55+
56+
let wkg_toml =
57+
wasm_pkg_core::config::Config::load_from_path(fixture.fixture_path.join("wkg.toml"))
58+
.await
59+
.expect("Should be able to load wkg.toml");
60+
let meta = wkg_toml.metadata.expect("Should have metadata");
61+
62+
assert_eq!(
63+
annotations
64+
.get("org.opencontainers.image.version")
65+
.expect("Should have version"),
66+
"0.2.0",
67+
"Version should be 0.2.0"
68+
);
69+
assert_eq!(
70+
annotations.get("org.opencontainers.image.description"),
71+
meta.description.as_ref(),
72+
"Description should match"
73+
);
74+
assert_eq!(
75+
annotations.get("org.opencontainers.image.licenses"),
76+
meta.license.as_ref(),
77+
"License should match"
78+
);
79+
assert_eq!(
80+
annotations.get("org.opencontainers.image.source"),
81+
meta.repository.as_ref(),
82+
"Source should match"
83+
);
84+
assert_eq!(
85+
annotations.get("org.opencontainers.image.url"),
86+
meta.homepage.as_ref(),
87+
"Name should match"
88+
);
89+
}
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
wkg.lock
2+
deps/

0 commit comments

Comments
 (0)