Repository: toml-rs/toml
Pull request #895 (2025-05-07 21:39:00)

fix(parse): Report missing '='


Patch:
diff --git a/crates/toml_parse/src/debug.rs b/crates/toml_parse/src/debug.rs
new file mode 100644
index 00000000..1f188641
--- /dev/null
+++ b/crates/toml_parse/src/debug.rs
@@ -0,0 +1,139 @@
+use crate::decode::Encoding;
+use crate::ErrorSink;
+use crate::Span;
+
+pub(crate) struct DebugDepth(core::sync::atomic::AtomicUsize);
+
+impl DebugDepth {
+    pub(crate) fn enter_unchecked(&self) -> usize {
+        self.0.fetch_add(1, core::sync::atomic::Ordering::SeqCst)
+    }
+
+    pub(crate) fn exit_unchecked(&self) {
+        let _ = self.0.fetch_sub(1, core::sync::atomic::Ordering::SeqCst);
+    }
+
+    pub(crate) fn depth(&self) -> usize {
+        self.0.load(core::sync::atomic::Ordering::SeqCst)
+    }
+}
+
+static DEBUG_DEPTH: DebugDepth = DebugDepth(core::sync::atomic::AtomicUsize::new(0));
+
+fn render_event(span: Span, text: &str, style: anstyle::Style) {
+    #![allow(unexpected_cfgs)] // HACK: fixed in newer versions
+    let depth = DEBUG_DEPTH.depth();
+    anstream::eprintln!("{:depth$}{style}{text}: {span:?}{style:#}", "");
+}
+
+pub(crate) struct DebugErrorSink<'s> {
+    sink: &'s mut dyn ErrorSink,
+}
+
+impl<'s> DebugErrorSink<'s> {
+    pub(crate) fn new(sink: &'s mut dyn ErrorSink) -> Self {
+        Self { sink }
+    }
+}
+
+impl ErrorSink for DebugErrorSink<'_> {
+    fn report_error(&mut self, error: crate::ParseError) {
+        render_event(
+            error.context,
+            &format!("{error:?}"),
+            anstyle::AnsiColor::Red.on_default(),
+        );
+        self.sink.report_error(error);
+    }
+}
+
+pub(crate) struct DebugEventReceiver<'r> {
+    receiver: &'r mut dyn crate::parser::EventReceiver,
+}
+
+impl<'r> DebugEventReceiver<'r> {
+    pub(crate) fn new(receiver: &'r mut dyn crate::parser::EventReceiver) -> Self {
+        Self { receiver }
+    }
+}
+
+impl crate::parser::EventReceiver for DebugEventReceiver<'_> {
+    fn std_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.std_table_open(span, error);
+        render_event(span, "[", anstyle::Style::new() | anstyle::Effects::DIMMED);
+        DEBUG_DEPTH.enter_unchecked();
+    }
+    fn std_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.std_table_close(span, error);
+        DEBUG_DEPTH.exit_unchecked();
+        render_event(span, "]", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn array_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.array_table_open(span, error);
+        render_event(span, "[[", anstyle::Style::new() | anstyle::Effects::DIMMED);
+        DEBUG_DEPTH.enter_unchecked();
+    }
+    fn array_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.array_table_close(span, error);
+        DEBUG_DEPTH.exit_unchecked();
+        render_event(span, "]]", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn inline_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) -> bool {
+        let allowed = self.receiver.inline_table_open(span, error);
+        render_event(span, "{", anstyle::Style::new() | anstyle::Effects::DIMMED);
+        DEBUG_DEPTH.enter_unchecked();
+        allowed
+    }
+    fn inline_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.inline_table_close(span, error);
+        DEBUG_DEPTH.exit_unchecked();
+        render_event(span, "}", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn array_open(&mut self, span: Span, error: &mut dyn ErrorSink) -> bool {
+        let allowed = self.receiver.array_open(span, error);
+        render_event(span, "[", anstyle::Style::new() | anstyle::Effects::DIMMED);
+        DEBUG_DEPTH.enter_unchecked();
+        allowed
+    }
+    fn array_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.array_close(span, error);
+        DEBUG_DEPTH.exit_unchecked();
+        render_event(span, "]", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn simple_key(&mut self, span: Span, encoding: Option<Encoding>, error: &mut dyn ErrorSink) {
+        self.receiver.simple_key(span, encoding, error);
+        render_event(span, "<key>", anstyle::AnsiColor::Magenta.on_default());
+    }
+    fn key_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.key_sep(span, error);
+        render_event(span, ".", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn key_val_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.key_val_sep(span, error);
+        render_event(span, "=", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn scalar(&mut self, span: Span, encoding: Option<Encoding>, error: &mut dyn ErrorSink) {
+        self.receiver.scalar(span, encoding, error);
+        render_event(span, "<scalar>", anstyle::AnsiColor::Green.on_default());
+    }
+    fn value_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.value_sep(span, error);
+        render_event(span, ",", anstyle::Style::new() | anstyle::Effects::DIMMED);
+    }
+    fn whitespace(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.whitespace(span, error);
+        render_event(span, "<whitespace>", anstyle::AnsiColor::Cyan.on_default());
+    }
+    fn comment(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.comment(span, error);
+        render_event(span, "<comment>", anstyle::AnsiColor::Cyan.on_default());
+    }
+    fn newline(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.newline(span, error);
+        render_event(span, "<newline>", anstyle::AnsiColor::Cyan.on_default());
+    }
+    fn error(&mut self, span: Span, error: &mut dyn ErrorSink) {
+        self.receiver.error(span, error);
+        render_event(span, "<error>", anstyle::AnsiColor::Red.on_default());
+    }
+}
diff --git a/crates/toml_parse/src/lexer/mod.rs b/crates/toml_parse/src/lexer/mod.rs
index c4121c94..145a40e6 100644
--- a/crates/toml_parse/src/lexer/mod.rs
+++ b/crates/toml_parse/src/lexer/mod.rs
@@ -26,10 +26,17 @@ pub struct Lexer<'i> {
 
 impl<'i> Lexer<'i> {
     pub(crate) fn new(input: &'i str) -> Self {
-        Lexer {
-            stream: Stream::new(input),
-            eof: false,
+        let mut stream = Stream::new(input);
+        if input.as_bytes().starts_with(BOM) {
+            let offset = BOM.len();
+            #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
+            unsafe {
+                stream.next_slice_unchecked(offset)
+            };
+            #[cfg(not(feature = "unsafe"))]
+            stream.next_slice(offset);
         }
+        Lexer { stream, eof: false }
     }
 
     #[cfg(feature = "alloc")]
@@ -92,6 +99,8 @@ impl Iterator for Lexer<'_> {
     }
 }
 
+const BOM: &[u8] = b"\xEF\xBB\xBF";
+
 pub(crate) type Stream<'i> = winnow::stream::LocatingSlice<&'i str>;
 
 /// Process an ASCII character token
diff --git a/crates/toml_parse/src/lib.rs b/crates/toml_parse/src/lib.rs
index 4c8cb1ac..b5e4d33b 100644
--- a/crates/toml_parse/src/lib.rs
+++ b/crates/toml_parse/src/lib.rs
@@ -14,6 +14,8 @@ extern crate alloc;
 #[macro_use]
 mod macros;
 
+#[cfg(feature = "unstable-debug")]
+pub(crate) mod debug;
 mod error;
 mod source;
 
diff --git a/crates/toml_parse/src/parser/document.rs b/crates/toml_parse/src/parser/document.rs
index b489b02f..6358a869 100644
--- a/crates/toml_parse/src/parser/document.rs
+++ b/crates/toml_parse/src/parser/document.rs
@@ -2,6 +2,10 @@ use winnow::stream::Stream as _;
 use winnow::stream::TokenSlice;
 
 use super::EventReceiver;
+#[cfg(feature = "unstable-debug")]
+use crate::debug::DebugErrorSink;
+#[cfg(feature = "unstable-debug")]
+use crate::debug::DebugEventReceiver;
 use crate::decode::Encoding;
 use crate::lexer::Token;
 use crate::lexer::TokenKind;
@@ -17,9 +21,13 @@ pub fn parse_document(
 ) {
     let mut tokens = TokenSlice::new(tokens);
     #[cfg(feature = "unstable-debug")]
-    let mut receiver = super::DebugEventReceiver::new(receiver);
+    let mut receiver = DebugEventReceiver::new(receiver);
     #[cfg(feature = "unstable-debug")]
     let receiver = &mut receiver;
+    #[cfg(feature = "unstable-debug")]
+    let mut error = DebugErrorSink::new(error);
+    #[cfg(feature = "unstable-debug")]
+    let error = &mut error;
     document(&mut tokens, receiver, error);
     eof(&mut tokens, receiver, error);
 }
@@ -28,9 +36,13 @@ pub fn parse_document(
 pub fn parse_key(tokens: &[Token], receiver: &mut dyn EventReceiver, error: &mut dyn ErrorSink) {
     let mut tokens = TokenSlice::new(tokens);
     #[cfg(feature = "unstable-debug")]
-    let mut receiver = super::DebugEventReceiver::new(receiver);
+    let mut receiver = DebugEventReceiver::new(receiver);
     #[cfg(feature = "unstable-debug")]
     let receiver = &mut receiver;
+    #[cfg(feature = "unstable-debug")]
+    let mut error = DebugErrorSink::new(error);
+    #[cfg(feature = "unstable-debug")]
+    let error = &mut error;
     key(&mut tokens, "key", receiver, error);
     eof(&mut tokens, receiver, error);
 }
@@ -43,9 +55,13 @@ pub fn parse_simple_key(
 ) {
     let mut tokens = TokenSlice::new(tokens);
     #[cfg(feature = "unstable-debug")]
-    let mut receiver = super::DebugEventReceiver::new(receiver);
+    let mut receiver = DebugEventReceiver::new(receiver);
     #[cfg(feature = "unstable-debug")]
     let receiver = &mut receiver;
+    #[cfg(feature = "unstable-debug")]
+    let mut error = DebugErrorSink::new(error);
+    #[cfg(feature = "unstable-debug")]
+    let error = &mut error;
     simple_key(&mut tokens, "key", receiver, error);
     eof(&mut tokens, receiver, error);
 }
@@ -54,9 +70,13 @@ pub fn parse_simple_key(
 pub fn parse_value(tokens: &[Token], receiver: &mut dyn EventReceiver, error: &mut dyn ErrorSink) {
     let mut tokens = TokenSlice::new(tokens);
     #[cfg(feature = "unstable-debug")]
-    let mut receiver = super::DebugEventReceiver::new(receiver);
+    let mut receiver = DebugEventReceiver::new(receiver);
     #[cfg(feature = "unstable-debug")]
     let receiver = &mut receiver;
+    #[cfg(feature = "unstable-debug")]
+    let mut error = DebugErrorSink::new(error);
+    #[cfg(feature = "unstable-debug")]
+    let error = &mut error;
     value(&mut tokens, receiver, error);
     eof(&mut tokens, receiver, error);
 }
@@ -373,6 +393,15 @@ fn on_expression_key<'i>(
     opt_whitespace(tokens, receiver, error);
 
     let Some(eq_token) = next_token_if(tokens, |k| matches!(k, TokenKind::Equals)) else {
+        if let Some(peek_token) = tokens.first() {
+            let span = peek_token.span().before();
+            error.report_error(ParseError {
+                context: span,
+                description: "expression",
+                expected: &[Expected::Literal("=")],
+                unexpected: span,
+            });
+        }
         ignore_to_newline(tokens, receiver, error);
         return;
     };
diff --git a/crates/toml_parse/src/parser/event.rs b/crates/toml_parse/src/parser/event.rs
index aa563d32..dba8a68e 100644
--- a/crates/toml_parse/src/parser/event.rs
+++ b/crates/toml_parse/src/parser/event.rs
@@ -383,110 +383,6 @@ impl EventReceiver for RecursionGuard<'_> {
     }
 }
 
-#[cfg(feature = "unstable-debug")]
-pub(crate) struct DebugEventReceiver<'r> {
-    receiver: &'r mut dyn EventReceiver,
-    indent: usize,
-}
-
-#[cfg(feature = "unstable-debug")]
-impl<'r> DebugEventReceiver<'r> {
-    pub(crate) fn new(receiver: &'r mut dyn EventReceiver) -> Self {
-        Self {
-            receiver,
-            indent: 0,
-        }
-    }
-
-    fn render_event(&mut self, span: Span, text: &str, style: anstyle::Style) {
-        #![allow(unexpected_cfgs)] // HACK: fixed in newer versions
-        let depth = self.indent;
-        anstream::eprintln!("{:depth$}{style}{text}: {span:?}{style:#}", "");
-    }
-}
-
-#[cfg(feature = "unstable-debug")]
-impl EventReceiver for DebugEventReceiver<'_> {
-    fn std_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.std_table_open(span, error);
-        self.render_event(span, "[", anstyle::Style::new() | anstyle::Effects::DIMMED);
-        self.indent += 1;
-    }
-    fn std_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.std_table_close(span, error);
-        self.indent -= 1;
-        self.render_event(span, "]", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn array_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.array_table_open(span, error);
-        self.render_event(span, "[[", anstyle::Style::new() | anstyle::Effects::DIMMED);
-        self.indent += 1;
-    }
-    fn array_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.array_table_close(span, error);
-        self.indent -= 1;
-        self.render_event(span, "]]", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn inline_table_open(&mut self, span: Span, error: &mut dyn ErrorSink) -> bool {
-        let allowed = self.receiver.inline_table_open(span, error);
-        self.render_event(span, "{", anstyle::Style::new() | anstyle::Effects::DIMMED);
-        self.indent += 1;
-        allowed
-    }
-    fn inline_table_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.inline_table_close(span, error);
-        self.indent -= 1;
-        self.render_event(span, "}", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn array_open(&mut self, span: Span, error: &mut dyn ErrorSink) -> bool {
-        let allowed = self.receiver.array_open(span, error);
-        self.render_event(span, "[", anstyle::Style::new() | anstyle::Effects::DIMMED);
-        self.indent += 1;
-        allowed
-    }
-    fn array_close(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.array_close(span, error);
-        self.indent -= 1;
-        self.render_event(span, "]", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn simple_key(&mut self, span: Span, encoding: Option<Encoding>, error: &mut dyn ErrorSink) {
-        self.receiver.simple_key(span, encoding, error);
-        self.render_event(span, "<key>", anstyle::AnsiColor::Magenta.on_default());
-    }
-    fn key_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.key_sep(span, error);
-        self.render_event(span, ".", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn key_val_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.key_val_sep(span, error);
-        self.render_event(span, "=", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn scalar(&mut self, span: Span, encoding: Option<Encoding>, error: &mut dyn ErrorSink) {
-        self.receiver.scalar(span, encoding, error);
-        self.render_event(span, "<scalar>", anstyle::AnsiColor::Green.on_default());
-    }
-    fn value_sep(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.value_sep(span, error);
-        self.render_event(span, ",", anstyle::Style::new() | anstyle::Effects::DIMMED);
-    }
-    fn whitespace(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.whitespace(span, error);
-        self.render_event(span, "<whitespace>", anstyle::AnsiColor::Cyan.on_default());
-    }
-    fn comment(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.comment(span, error);
-        self.render_event(span, "<comment>", anstyle::AnsiColor::Cyan.on_default());
-    }
-    fn newline(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.newline(span, error);
-        self.render_event(span, "<newline>", anstyle::AnsiColor::Cyan.on_default());
-    }
-    fn error(&mut self, span: Span, error: &mut dyn ErrorSink) {
-        self.receiver.error(span, error);
-        self.render_event(span, "<error>", anstyle::AnsiColor::Red.on_default());
-    }
-}
-
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 pub struct Event {
     kind: EventKind,
diff --git a/crates/toml_parse/src/parser/mod.rs b/crates/toml_parse/src/parser/mod.rs
index 02777339..15142700 100644
--- a/crates/toml_parse/src/parser/mod.rs
+++ b/crates/toml_parse/src/parser/mod.rs
@@ -9,6 +9,3 @@ pub use event::Event;
 pub use event::EventKind;
 pub use event::EventReceiver;
 pub use event::RecursionGuard;
-
-#[cfg(feature = "unstable-debug")]
-pub(crate) use event::DebugEventReceiver;


Test patch:
diff --git a/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_bom.txt b/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_bom.txt
index 70cde163..16d5f2e7 100644
--- a/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_bom.txt
+++ b/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_bom.txt
@@ -1,11 +1,6 @@
 EventResults {
     input: "\u{feff}\n[package]\nname = \"foo\"\nversion = \"0.0.1\"\nauthors = []\n",
     events: [
-        Event {
-            kind: SimpleKey,
-            encoding: None,
-            span: 0..3,
-        },
         Event {
             kind: Newline,
             encoding: None,
@@ -131,63 +126,5 @@ EventResults {
             span: 57..58,
         },
     ],
-    errors: [
-        ParseError {
-            context: 0..3,
-            description: "unquoted string",
-            expected: [
-                Description(
-                    "letters",
-                ),
-                Description(
-                    "numbers",
-                ),
-                Literal(
-                    "-",
-                ),
-                Literal(
-                    "_",
-                ),
-            ],
-            unexpected: 0..0,
-        },
-        ParseError {
-            context: 0..3,
-            description: "unquoted string",
-            expected: [
-                Description(
-                    "letters",
-                ),
-                Description(
-                    "numbers",
-                ),
-                Literal(
-                    "-",
-                ),
-                Literal(
-                    "_",
-                ),
-            ],
-            unexpected: 1..1,
-        },
-        ParseError {
-            context: 0..3,
-            description: "unquoted string",
-            expected: [
-                Description(
-                    "letters",
-                ),
-                Description(
-                    "numbers",
-                ),
-                Literal(
-                    "-",
-                ),
-                Literal(
-                    "_",
-                ),
-            ],
-            unexpected: 2..2,
-        },
-    ],
+    errors: [],
 }
diff --git a/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_invalid.txt b/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_invalid.txt
index 9e8dbdf8..af07af9f 100644
--- a/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_invalid.txt
+++ b/crates/toml_parse/tests/snapshots/testsuite__parse_document__document_invalid.txt
@@ -55,6 +55,16 @@ EventResults {
         },
     ],
     errors: [
+        ParseError {
+            context: 37..37,
+            description: "expression",
+            expected: [
+                Literal(
+                    "=",
+                ),
+            ],
+            unexpected: 37..37,
+        },
         ParseError {
             context: 36..37,
             description: "unquoted string",
diff --git a/crates/toml_parse/tests/testsuite/main.rs b/crates/toml_parse/tests/testsuite/main.rs
index e6fae45d..3fb95756 100644
--- a/crates/toml_parse/tests/testsuite/main.rs
+++ b/crates/toml_parse/tests/testsuite/main.rs
@@ -104,12 +104,26 @@ impl<'i> EventResults<'i> {
         assert_data_eq!(self.to_debug(), expected);
         if !self.events.is_empty() {
             let spans = self.events.iter().map(|t| t.span()).collect::<Vec<_>>();
-            assert_eq!(spans.first().unwrap().start(), 0);
-            assert_eq!(spans.last().unwrap().end(), self.input.len());
+            if !self.input.as_bytes().starts_with(BOM) {
+                assert_eq!(
+                    spans.first().unwrap().start(),
+                    0,
+                    "first span needs to start at 0"
+                );
+            }
+            assert_eq!(
+                spans.last().unwrap().end(),
+                self.input.len(),
+                "last span needs to be at the end"
+            );
             for i in 0..(spans.len() - 1) {
                 let current = &spans[i];
                 let next = &spans[i + 1];
-                assert_eq!(current.end(), next.start());
+                assert_eq!(
+                    current.end(),
+                    next.start(),
+                    "events must not have gaps in spans"
+                );
             }
         }
         if self.events.iter().any(|e| e.kind() == EventKind::Error) {
@@ -117,3 +131,5 @@ impl<'i> EventResults<'i> {
         }
     }
 }
+
+const BOM: &[u8] = b"\xEF\xBB\xBF";