Add a package declaration to the grammar.

This commit adds a `package` directive to the grammar to match what is present
in WIT.

As a result, the `--package` command line argument to the `parse` command was
removed.

Additionally, lexing of semver is relaxed so that invalid semantic versions can
be detected when building the AST.

Author: peterhuene

Cargo.lock

@@ -40,7 +40,7 @@ wasmparser = "0.116.1"
 wit-component = "0.18.0"
 anyhow = "1.0.75"
 clap = { version = "4.4.7", features = ["derive"] }
-semver = "1.0.20"
+semver = { version = "1.0.20", features = ["serde"] }
 pretty_env_logger = "0.5.0"
 log = "0.4.20"
 tokio = { version = "1.33.0", default-features = false, features = ["macros", "rt-multi-thread"] }

Cargo.toml

@@ -3,17 +3,19 @@
 The current WAC grammar:
 
 ```ebnf
-document  ::= statement*
-statement ::= import-statement 
+document  ::= package-decl statement*
+statement ::= import-statement
             | type-statement
             | let-statement
             | export-statement
 
+package-decl ::= `package` package-name `;`
+package-name ::= id (':' id)+ ('@' version)?
+version      ::= <SEMVER>
+
 import-statement ::= 'import' id ('with' string)? ':' import-type ';'
 import-type      ::= package-path | func-type | inline-interface | id
-package-path     ::= package-name ('/' id)+ ('@' package-version)?
-package-name     ::= id (':' id)+
-package-version  ::= <SEMVER>
+package-path     ::= id (':' id)+ ('/' id)+ ('@' version)?
 
 type-statement      ::= interface-decl | world-decl | type-decl
 interface-decl      ::= 'interface' id '{' interface-item* '}'

README.md

@@ -113,6 +113,14 @@ pub enum Error<'a> {
         /// The span of the empty type.
         span: Span<'a>,
     },
+    /// An invalid semantic version was encountered.
+    #[error("`{version}` is not a valid semantic version")]
+    InvalidVersion {
+        /// The invalid version.
+        version: &'a str,
+        /// The span of the version.
+        span: Span<'a>,
+    },
 }
 
 impl Spanned for Error<'_> {
@@ -122,7 +130,8 @@ impl Spanned for Error<'_> {
             | Error::Expected { span, .. }
             | Error::ExpectedEither { span, .. }
             | Error::ExpectedMultiple { span, .. }
-            | Error::EmptyType { span, .. } => *span,
+            | Error::EmptyType { span, .. }
+            | Error::InvalidVersion { span, .. } => *span,
         }
     }
 }
@@ -313,6 +322,10 @@ pub struct Document<'a> {
     /// The path to the document, used for error reporting.
     #[serde(skip)]
     pub path: &'a Path,
+    /// The doc comments for the package.
+    pub docs: Vec<DocComment<'a>>,
+    /// The package name of the document.
+    pub package: PackageName<'a>,
     /// The statements in the document.
     pub statements: Vec<Statement<'a>>,
 }
@@ -323,14 +336,25 @@ impl<'a> Document<'a> {
     /// The given path is used for error reporting.
     pub fn parse(source: &'a str, path: &'a Path) -> ParseResult<'a, Self> {
         let mut lexer = Lexer::new(source).map_err(Error::from)?;
-        let mut statements: Vec<Statement> = Default::default();
 
+        let docs = Parse::parse(&mut lexer)?;
+
+        parse_token(&mut lexer, Token::PackageKeyword)?;
+        let package = Parse::parse(&mut lexer)?;
+        parse_token(&mut lexer, Token::Semicolon)?;
+
+        let mut statements: Vec<Statement> = Default::default();
         while lexer.peek().is_some() {
             statements.push(Parse::parse(&mut lexer)?);
         }
 
         assert!(lexer.next().is_none(), "expected all tokens to be consumed");
-        Ok(Self { path, statements })
+        Ok(Self {
+            path,
+            docs,
+            package,
+            statements,
+        })
     }
 }
 
@@ -468,6 +492,8 @@ mod test {
     fn document_roundtrip() {
         let document = Document::parse(
             r#"/* ignore me */
+/// Doc comment for the package!
+package test:foo:bar@1.0.0;
 /// Doc comment #1!
 import foo: foo:bar/baz;
 /// Doc comment #2!
@@ -499,7 +525,10 @@ export x with "foo";
 
         assert_eq!(
             document.to_string(),
-            r#"/// Doc comment #1!
+            r#"/// Doc comment for the package!
+package test:foo:bar@1.0.0;
+
+/// Doc comment #1!
 import foo: foo:bar/baz;
 
 /// Doc comment #2!

crates/wac-parser/src/ast.rs

@@ -1,8 +1,9 @@
 use super::{
-    display, parse_optional, parse_token, DocComment, FuncType, Ident, InlineInterface, Lookahead,
-    Parse, ParseResult, Peek,
+    display, parse_optional, parse_token, DocComment, Error, FuncType, Ident, InlineInterface,
+    Lookahead, Parse, ParseResult, Peek,
 };
 use crate::lexer::{Lexer, Span, Token};
+use semver::Version;
 use serde::Serialize;
 
 /// Represents an import statement in the AST.
@@ -82,7 +83,7 @@ pub struct PackagePath<'a> {
     /// The path segments.
     pub segments: &'a str,
     /// The optional version of the package.
-    pub version: Option<&'a str>,
+    pub version: Option<Version>,
 }
 
 impl<'a> PackagePath<'a> {
@@ -112,7 +113,15 @@ impl<'a> Parse<'a> for PackagePath<'a> {
         let at = s.find('@');
         let name = &s[..slash];
         let segments = &s[slash + 1..at.unwrap_or(slash + s.len() - name.len())];
-        let version = at.map(|at| &s[at + 1..]);
+        let version = at
+            .map(|at| {
+                let version = &s[at + 1..];
+                version.parse().map_err(|_| Error::InvalidVersion {
+                    version,
+                    span: Span::from_span(span.source(), &(span.start + at + 1..span.end)),
+                })
+            })
+            .transpose()?;
 
         Ok(Self {
             span,
@@ -135,15 +144,32 @@ impl Peek for PackagePath<'_> {
 pub struct PackageName<'a> {
     /// The name of the package.
     pub name: &'a str,
+    /// The optional version of the package.
+    pub version: Option<Version>,
     /// The span of the package name,
     pub span: Span<'a>,
 }
 
 impl<'a> Parse<'a> for PackageName<'a> {
     fn parse(lexer: &mut Lexer<'a>) -> ParseResult<'a, Self> {
         let span = parse_token(lexer, Token::PackageName)?;
-        let name = span.as_str();
-        Ok(Self { name, span })
+        let s = span.as_str();
+        let at = s.find('@');
+        let name = at.map(|at| &s[..at]).unwrap_or(s);
+        let version = at
+            .map(|at| {
+                let version = &s[at + 1..];
+                version.parse().map_err(|_| Error::InvalidVersion {
+                    version,
+                    span: Span::from_span(span.source(), &(span.start + at + 1..span.end)),
+                })
+            })
+            .transpose()?;
+        Ok(Self {
+            name,
+            version,
+            span,
+        })
     }
 }
 

crates/wac-parser/src/ast/import.rs

@@ -148,7 +148,7 @@ fn detect_invalid_input(source: &str) -> Result<(), (Error, Span)> {
 #[logos(skip r"[ \t\n\f]+")]
 #[logos(subpattern id = r"%?[a-z][a-z0-9]*(-[a-z][a-z0-9]*)*")]
 #[logos(subpattern package_name = r"(?&id)(:(?&id))+")]
-#[logos(subpattern semver = r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-((0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?")]
+#[logos(subpattern semver = r"[0-9a-zA-Z-\.\+]+")]
 pub enum Token {
     /// A comment.
     #[regex(r"//[^\n]*", logos::skip)]
@@ -167,7 +167,7 @@ pub enum Token {
     String,
 
     /// A package name.
-    #[regex(r"(?&id)(:(?&id))+")]
+    #[regex(r"(?&package_name)(@(?&semver))?")]
     PackageName,
 
     /// A package path with optional semantic version.
@@ -285,6 +285,9 @@ pub enum Token {
     /// The `as` keyword.
     #[token("as")]
     AsKeyword,
+    /// The `package` keyword.
+    #[token("package")]
+    PackageKeyword,
 
     /// The `;` symbol.
     #[token(";")]
@@ -387,6 +390,7 @@ impl fmt::Display for Token {
             Token::UseKeyword => write!(f, "`use` keyword"),
             Token::IncludeKeyword => write!(f, "`include` keyword"),
             Token::AsKeyword => write!(f, "`as` keyword"),
+            Token::PackageKeyword => write!(f, "`package` keyword"),
             Token::Semicolon => write!(f, "`;`"),
             Token::OpenBrace => write!(f, "`{{`"),
             Token::CloseBrace => write!(f, "`}}`"),

crates/wac-parser/src/lexer.rs

@@ -27,6 +27,14 @@ impl<W: Write> DocumentPrinter<W> {
 
     /// Prints the given document.
     pub fn document(&mut self, doc: &Document) -> std::fmt::Result {
+        self.docs(&doc.docs)?;
+        writeln!(
+            self.writer,
+            "package {package};",
+            package = doc.package.span.as_str()
+        )?;
+        self.newline()?;
+
         for (i, statement) in doc.statements.iter().enumerate() {
             if i > 0 {
                 self.newline()?;

crates/wac-parser/src/printer.rs

@@ -9,6 +9,7 @@ use crate::{
 use anyhow::Context;
 use id_arena::{Arena, Id};
 use indexmap::{IndexMap, IndexSet};
+use semver::Version;
 use serde::{Serialize, Serializer};
 use std::{
     collections::{hash_map, HashMap},
@@ -594,7 +595,7 @@ pub trait PackageResolver {
     /// Resolves a package name to the package bytes.
     ///
     /// Returns `Ok(None)` if the package could not be found.
-    fn resolve(&self, name: &str) -> anyhow::Result<Option<Vec<u8>>>;
+    fn resolve(&self, name: &str, version: Option<&Version>) -> anyhow::Result<Option<Vec<u8>>>;
 }
 
 /// Used to resolve packages from the file system.
@@ -620,7 +621,7 @@ impl Default for FileSystemPackageResolver {
 }
 
 impl PackageResolver for FileSystemPackageResolver {
-    fn resolve(&self, name: &str) -> anyhow::Result<Option<Vec<u8>>> {
+    fn resolve(&self, name: &str, _version: Option<&Version>) -> anyhow::Result<Option<Vec<u8>>> {
         let path = if let Some(path) = self.overrides.get(name) {
             if !path.exists() {
                 anyhow::bail!(
@@ -840,6 +841,8 @@ impl fmt::Display for FuncKind {
 pub struct ResolvedDocument {
     /// The name of the package being resolved.
     pub package: String,
+    /// The version of the package being resolved.
+    pub version: Option<Version>,
     /// The definitions in the resolution.
     pub definitions: Definitions,
     /// The items in the resolution.
@@ -860,7 +863,6 @@ impl ResolvedDocument {
     /// Creates a new resolved document from the given document.
     pub fn new<'a>(
         document: &'a ast::Document<'a>,
-        package: impl Into<String>,
         resolver: Option<Box<dyn PackageResolver>>,
     ) -> ResolutionResult<'a, Self> {
         let mut scopes = Arena::new();
@@ -870,7 +872,8 @@ impl ResolvedDocument {
         });
 
         let mut resolution = ResolvedDocument {
-            package: package.into(),
+            package: document.package.name.to_owned(),
+            version: document.package.version.clone(),
             definitions: Default::default(),
             items: Default::default(),
             imports: Default::default(),
@@ -2069,6 +2072,7 @@ impl ResolvedDocument {
         &mut self,
         state: &'b mut ResolutionState<'a>,
         name: &'a str,
+        version: Option<&Version>,
         span: Span<'a>,
     ) -> ResolutionResult<'a, &'b Package> {
         match state.packages.entry(name.to_owned()) {
@@ -2078,7 +2082,7 @@ impl ResolvedDocument {
                 match state
                     .resolver
                     .as_deref()
-                    .and_then(|r| r.resolve(name).transpose())
+                    .and_then(|r| r.resolve(name, version).transpose())
                     .transpose()
                     .map_err(|e| Error::PackageResolutionFailure {
                         name,
@@ -2110,7 +2114,12 @@ impl ResolvedDocument {
             return self.resolve_local_export(state, path);
         }
 
-        let pkg = self.resolve_package(state, path.name, path.package_name_span())?;
+        let pkg = self.resolve_package(
+            state,
+            path.name,
+            path.version.as_ref(),
+            path.package_name_span(),
+        )?;
 
         let mut current = 0;
         let mut parent_ty = None;
@@ -2257,7 +2266,12 @@ impl ResolvedDocument {
         state: &mut ResolutionState<'a>,
         expr: &'a ast::NewExpr<'a>,
     ) -> ResolutionResult<'a, ItemId> {
-        let pkg = self.resolve_package(state, expr.package.name, expr.package.span)?;
+        let pkg = self.resolve_package(
+            state,
+            expr.package.name,
+            expr.package.version.as_ref(),
+            expr.package.span,
+        )?;
         let ty = pkg.ty;
         let require_all = !expr.ellipsis;
 

crates/wac-parser/src/resolution.rs

@@ -1,3 +1,5 @@
+package test:comp;
+
 /// Export an item (default name)
 export e;