feat(config): Adds additional configuration option for specifying metadata

Adds support for an additional configuration for namespaces and packages
so a well-known is not necessarily required

Signed-off-by: Taylor Thomas <taylor@cosmonic.com>

Author: thomastaylor312

README.md

@@ -69,11 +69,20 @@ default_registry = "acme.registry.com"
 [namespace_registries]
 wasi = "wasi.dev"
 example = "example.com"
+# An example of providing your own registry mapping. For large and/or public registries, we
+# recommend creating a well-known metadata file that can be used to determine the registry to use
+# (see the section on "metadata" below). But many times you might want to override mappings or
+# provide something that is used by a single team. The registry name does not matter, but must be
+# parsable to URL authority. This name is purely used for mapping to registry config and isn't
+# actually used as a URL when metadata is provided 
+another = { registry = "another", metadata = { preferredProtocol = "oci", "oci" = {registry = "ghcr.io", namespacePrefix = "webassembly/" } } }
 
 # This overrides the default registry for a specific package. This is useful for cases where a 
 # package is published to multiple registries. 
 [package_registry_overrides]
 "example:foo" = "example.com"
+# Same as namespace_registries above, but for a specific package.
+"example:bar" = { registry = "another", metadata = { preferredProtocol = "oci", "oci" = {registry = "ghcr.io", namespacePrefix = "webassembly/" } } }
 
 # This section contains a mapping of registries to their configuration. There are currently 3
 # supported types of registries: "oci", "warg", and "local". The "oci" type is the default. The
@@ -117,14 +126,20 @@ root = "/a/path"
 # config_file = "/a/path"
 [registry."example.com".warg]
 config_file = "/a/path"
+
+# Configuration for the "another" registry defined above.
+[registry."another".oci]
+auth = { username = "open", password = "sesame" }
 ```
 
 ### Well-known metadata
 
-The `wkg` tool and libraries expect a `registry.json` file to be present at a specific location to
-indicate to the tooling where the components are stored. If a registry was `example.com`, then the
-tooling will attempt to find a `registry.json` file at
-`https://example.com/.well-known/wasm-pkg/registry.json`. 
+For well-used or public registries, we recommend creating a well-known metadata file that is used by
+the tool chain to simplify configuration and indicate to a client which protocols and mappings to
+use (although this can be set directly in config as well). The `wkg` tool and libraries expect a
+`registry.json` file to be present at a specific location to indicate to the tooling where the
+components are stored. For example, if a registry was `example.com`, then the tooling will attempt
+to find a `registry.json` file at `https://example.com/.well-known/wasm-pkg/registry.json`. 
 
 A full example of what this `registry.json` file should look like is below:
 

crates/wasm-pkg-client/src/lib.rs

@@ -46,7 +46,7 @@ use tokio::io::AsyncSeekExt;
 use tokio::sync::RwLock;
 use tokio_util::io::SyncIoBridge;
 pub use wasm_pkg_common::{
-    config::Config,
+    config::{Config, CustomConfig, RegistryMapping},
     digest::ContentDigest,
     metadata::RegistryMetadata,
     package::{PackageRef, Version},
@@ -191,6 +191,7 @@ impl Client {
         package: &PackageRef,
         registry_override: Option<Registry>,
     ) -> Result<Arc<InnerClient>, Error> {
+        let is_override = registry_override.is_some();
         let registry = if let Some(registry) = registry_override {
             registry
         } else {
@@ -212,7 +213,25 @@ impl Client {
 
             // Skip fetching metadata for "local" source
             let should_fetch_meta = registry_config.default_backend() != Some("local");
-            let registry_meta = if should_fetch_meta {
+            let maybe_metadata = self
+                .config
+                .namespace_registry(package.namespace())
+                .and_then(|meta| {
+                    // If the overriden registry matches the registry we are trying to resolve, we
+                    // should use the metadata, otherwise we'll need to fetch the metadata from the
+                    // registry
+                    match (meta, is_override) {
+                        (RegistryMapping::Custom(custom), true) if custom.registry == registry => {
+                            Some(custom.metadata.clone())
+                        }
+                        (RegistryMapping::Custom(custom), false) => Some(custom.metadata.clone()),
+                        _ => None,
+                    }
+                });
+
+            let registry_meta = if let Some(meta) = maybe_metadata {
+                meta
+            } else if should_fetch_meta {
                 RegistryMetadata::fetch_or_default(&registry).await
             } else {
                 RegistryMetadata::default()

crates/wasm-pkg-client/src/oci/config.rs

@@ -159,6 +159,10 @@ fn serialize_secret<S: Serializer>(
 
 #[cfg(test)]
 mod tests {
+    use wasm_pkg_common::config::RegistryMapping;
+
+    use crate::oci::OciRegistryMetadata;
+
     use super::*;
 
     #[test]
@@ -241,4 +245,30 @@ mod tests {
             "Password should be set to the right value"
         );
     }
+
+    #[test]
+    fn test_custom_namespace_config() {
+        let toml_config = toml::toml! {
+            [namespace_registries]
+            test = { registry = "localhost:1234", metadata = { preferredProtocol = "oci", "oci" = { registry = "ghcr.io", namespacePrefix = "webassembly/" } } }
+        };
+
+        let cfg = wasm_pkg_common::config::Config::from_toml(&toml_config.to_string())
+            .expect("Should be able to load config");
+
+        let ns_config = cfg
+            .namespace_registry(&"test".parse().unwrap())
+            .expect("Should have a namespace config");
+        let custom = match ns_config {
+            RegistryMapping::Custom(c) => c,
+            _ => panic!("Should have a custom namespace config"),
+        };
+        let map: OciRegistryMetadata = custom
+            .metadata
+            .protocol_config("oci")
+            .expect("Should be able to deserialize config")
+            .expect("protocol config should be present");
+        assert_eq!(map.namespace_prefix, Some("webassembly/".to_string()));
+        assert_eq!(map.registry, Some("ghcr.io".to_string()));
+    }
 }

crates/wasm-pkg-client/src/warg/config.rs

@@ -116,6 +116,10 @@ fn serialize_secret<S: Serializer>(
 
 #[cfg(test)]
 mod tests {
+    use wasm_pkg_common::config::RegistryMapping;
+
+    use crate::warg::WargRegistryMetadata;
+
     use super::*;
 
     #[tokio::test]
@@ -178,4 +182,33 @@ mod tests {
             "Signing key should be set to the right value"
         );
     }
+
+    #[test]
+    fn test_custom_namespace_config() {
+        let toml_config = toml::toml! {
+            [namespace_registries]
+            test = { registry = "localhost:1234", metadata = { preferredProtocol = "warg", "warg" = { url = "http://localhost:1234" } } }
+        };
+
+        let cfg = wasm_pkg_common::config::Config::from_toml(&toml_config.to_string())
+            .expect("Should be able to load config");
+
+        let ns_config = cfg
+            .namespace_registry(&"test".parse().unwrap())
+            .expect("Should have a namespace config");
+        let custom = match ns_config {
+            RegistryMapping::Custom(c) => c,
+            _ => panic!("Should have a custom namespace config"),
+        };
+        let map: WargRegistryMetadata = custom
+            .metadata
+            .protocol_config("warg")
+            .expect("Should be able to deserialize config")
+            .expect("protocol config should be present");
+        assert_eq!(
+            map.url,
+            Some("http://localhost:1234".into()),
+            "URL should be set to the right value"
+        );
+    }
 }

crates/wasm-pkg-client/tests/e2e.rs

@@ -59,3 +59,28 @@ async fn publish_and_fetch_smoke_test() {
         .expect("Failed to read fixture");
     assert_eq!(content, expected_content);
 }
+
+// Simple smoke test to make sure the custom metadata section is parsed and used correctly. Down the
+// line we might want to just push a thing to a local registry and then fetch it, but for now we'll
+// just use the bytecodealliance registry.
+#[tokio::test]
+async fn fetch_with_custom_config() {
+    let toml_config = toml::toml! {
+        [namespace_registries]
+        wasi = { registry = "fake.com:1234", metadata = { preferredProtocol = "oci", "oci" = {registry = "ghcr.io", namespacePrefix = "bytecodealliance/wasm-pkg/" } } }
+    };
+
+    let conf = Config::from_toml(&toml_config.to_string()).expect("Failed to parse config");
+    let client = Client::new(conf);
+
+    // Try listing all versions of the wasi package and make sure it doesn't fail
+    let package = "wasi:http".parse().unwrap();
+    let versions = client
+        .list_all_versions(&package)
+        .await
+        .expect("Should be able to list versions with custom config");
+    assert!(
+        !versions.is_empty(),
+        "Should be able to list versions with custom config"
+    );
+}

crates/wasm-pkg-common/src/config.rs

@@ -6,7 +6,9 @@ use std::{
 
 use serde::{Deserialize, Serialize};
 
-use crate::{label::Label, package::PackageRef, registry::Registry, Error};
+use crate::{
+    label::Label, metadata::RegistryMetadata, package::PackageRef, registry::Registry, Error,
+};
 
 mod toml;
 
@@ -23,13 +25,35 @@ const DEFAULT_FALLBACK_NAMESPACE_REGISTRIES: &[(&str, &str)] =
 #[serde(into = "toml::TomlConfig")]
 pub struct Config {
     default_registry: Option<Registry>,
-    namespace_registries: HashMap<Label, Registry>,
-    package_registry_overrides: HashMap<PackageRef, Registry>,
+    namespace_registries: HashMap<Label, RegistryMapping>,
+    package_registry_overrides: HashMap<PackageRef, RegistryMapping>,
     // Note: these are only used for hard-coded defaults currently
     fallback_namespace_registries: HashMap<Label, Registry>,
     registry_configs: HashMap<Registry, RegistryConfig>,
 }
 
+/// Possible options for namespace configuration.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum RegistryMapping {
+    /// Use the given registry address (this will fetch the well-known registry metadata from the given hostname).
+    Registry(Registry),
+    /// Use custom configuration for reaching a registry
+    Custom(CustomConfig),
+}
+
+/// Custom registry configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CustomConfig {
+    /// A valid name for the registry. This still must be a valid [`Registry`] in that it should
+    /// look like a valid hostname. When doing custom configuration however, this is just used as a
+    /// key to identify the configuration for this namespace
+    pub registry: Registry,
+    /// The metadata for the registry. This is used to determine the protocol to use for the
+    /// registry as well as mapping information for the registry.
+    pub metadata: RegistryMetadata,
+}
+
 impl Default for Config {
     fn default() -> Self {
         let fallback_namespace_registries = DEFAULT_FALLBACK_NAMESPACE_REGISTRIES
@@ -150,10 +174,20 @@ impl Config {
     /// - The default registry
     /// - Hard-coded fallbacks for certain well-known namespaces
     pub fn resolve_registry(&self, package: &PackageRef) -> Option<&Registry> {
-        if let Some(reg) = self.package_registry_overrides.get(package) {
+        if let Some(RegistryMapping::Registry(reg)) = self.package_registry_overrides.get(package) {
             Some(reg)
-        } else if let Some(reg) = self.namespace_registries.get(package.namespace()) {
+        } else if let Some(RegistryMapping::Custom(custom)) =
+            self.package_registry_overrides.get(package)
+        {
+            Some(&custom.registry)
+        } else if let Some(RegistryMapping::Registry(reg)) =
+            self.namespace_registries.get(package.namespace())
+        {
             Some(reg)
+        } else if let Some(RegistryMapping::Custom(custom)) =
+            self.namespace_registries.get(package.namespace())
+        {
+            Some(&custom.registry)
         } else if let Some(reg) = self.default_registry.as_ref() {
             Some(reg)
         } else if let Some(reg) = self.fallback_namespace_registries.get(package.namespace()) {
@@ -179,25 +213,29 @@ impl Config {
     ///
     /// Does not fall back to the default registry; see
     /// [`Self::resolve_registry`].
-    pub fn namespace_registry(&self, namespace: &Label) -> Option<&Registry> {
+    pub fn namespace_registry(&self, namespace: &Label) -> Option<&RegistryMapping> {
         self.namespace_registries.get(namespace)
     }
 
     /// Sets a registry for the given namespace.
-    pub fn set_namespace_registry(&mut self, namespace: Label, registry: Registry) {
+    pub fn set_namespace_registry(&mut self, namespace: Label, registry: RegistryMapping) {
         self.namespace_registries.insert(namespace, registry);
     }
 
     /// Returns a registry override configured for the given package.
     ///
     /// Does not fall back to namespace or default registries; see
     /// [`Self::resolve_registry`].
-    pub fn package_registry_override(&self, package: &PackageRef) -> Option<&Registry> {
+    pub fn package_registry_override(&self, package: &PackageRef) -> Option<&RegistryMapping> {
         self.package_registry_overrides.get(package)
     }
 
     /// Sets a registry override for the given package.
-    pub fn set_package_registry_override(&mut self, package: PackageRef, registry: Registry) {
+    pub fn set_package_registry_override(
+        &mut self,
+        package: PackageRef,
+        registry: RegistryMapping,
+    ) {
         self.package_registry_overrides.insert(package, registry);
     }