Improve error message for missing delimiter at EOF (#1590)

* Improve error message for missing delimiter at EOF

Due to the way source maps worked the span would point to the wrong file
if the marker was pointing at the end of a file. While this still isn't
necessarily a perfect error message it should be a significant
improvement over before by actually pointing to the incorrect file.

Closes #1583

* Update test expectation

* Fix span calculation for eof

* Re-bless test output

Author: alexcrichton

crates/wit-parser/src/ast.rs

@@ -1385,10 +1385,7 @@ fn err_expected(
             format!("expected {}, found {}", expected, token.describe()),
         ),
         None => Error::new(
-            Span {
-                start: u32::try_from(tokens.input().len()).unwrap(),
-                end: u32::try_from(tokens.input().len()).unwrap(),
-            },
+            tokens.eof_span(),
             format!("expected {}, found eof", expected),
         ),
     }
@@ -1519,7 +1516,13 @@ impl SourceMap {
     /// interfaces and worlds defined together. Note that each file has its own
     /// personal namespace, however, for top-level `use` and such.
     pub fn push(&mut self, path: &Path, contents: impl Into<String>) {
-        let contents = contents.into();
+        let mut contents = contents.into();
+        // Guarantee that there's at least one character in these contents by
+        // appending a single newline to the end. This is excluded from
+        // tokenization below so it's only here to ensure that spans which point
+        // one byte beyond the end of a file (eof) point to the same original
+        // file.
+        contents.push('\n');
         let new_offset = self.offset + u32::try_from(contents.len()).unwrap();
         self.sources.push(Source {
             offset: self.offset,
@@ -1537,7 +1540,9 @@ impl SourceMap {
             srcs.sort_by_key(|src| &src.path);
             for src in srcs {
                 let mut tokens = Tokenizer::new(
-                    &src.contents,
+                    // chop off the forcibly appended `\n` character when
+                    // passing through the source to get tokenized.
+                    &src.contents[..src.contents.len() - 1],
                     src.offset,
                     self.require_semicolons,
                     self.require_f32_f64,

crates/wit-parser/src/ast/lex.rs

@@ -162,10 +162,6 @@ impl<'a> Tokenizer<'a> {
         Ok(())
     }
 
-    pub fn input(&self) -> &'a str {
-        self.input
-    }
-
     pub fn get_span(&self, span: Span) -> &'a str {
         let start = usize::try_from(span.start - self.span_offset).unwrap();
         let end = usize::try_from(span.end - self.span_offset).unwrap();
@@ -395,6 +391,11 @@ impl<'a> Tokenizer<'a> {
             _ => false,
         }
     }
+
+    pub fn eof_span(&self) -> Span {
+        let end = self.span_offset + u32::try_from(self.input.len()).unwrap();
+        Span { start: end, end }
+    }
 }
 
 impl<'a> Iterator for CrlfFold<'a> {

crates/wit-parser/tests/ui/parse-fail/multi-file-missing-delimiter.wit.result

@@ -0,0 +1,9 @@
+failed to resolve directory while parsing WIT for path [tests/ui/parse-fail/multi-file-missing-delimiter]
+
+Caused by:
+    0: failed to parse package: tests/ui/parse-fail/multi-file-missing-delimiter
+    1: expected `type`, `resource` or `func`, found eof
+            --> tests/ui/parse-fail/multi-file-missing-delimiter/observe.wit:6:1
+             |
+           6 | 
+             | ^

crates/wit-parser/tests/ui/parse-fail/multi-file-missing-delimiter/observe.wit

@@ -0,0 +1,5 @@
+interface observe {
+  record bar {
+    a : string,
+  }
+  // NB: intentionally missing a delimiter at the end of the file

crates/wit-parser/tests/ui/parse-fail/multi-file-missing-delimiter/world.wit

@@ -0,0 +1,5 @@
+package qux:qaz@2.0.0;
+
+world platform {
+  import observe;
+}