Repository: softdevteam/grmtools
Pull request #564: Add line/column info to nimbleparses printing of `HeaderError`.
Author: login: ratmice
Created: 2025-05-11T22:42:16Z | Merged: 2025-05-16T09:17:36Z

In the process replace some usage of `Option` with `OnceCell`.

It also restores some spanned error printing that was lost in commit 277ae03622cb956455d6e98ee5e7efa4f71b3855 when `Location` was in `YaccGrammarError`.

This is a follow up to PR #561 in that PR the errors when run through nimbleparse looked like:

```console
Error(s) parsing `%grmtools` section:
 Unxpected token: '*', perhaps this is a glob, in which case it requires string quoting.
```

After this patch they include source context information,

```
[Error] parsing the `%grmtools` section: in foo.y
    3|     test_files: *.test,
                        Unxpected token: '*', perhaps this is a glob, in which case it requires string quoting.
[Error] parsing the `%grmtools` section: in bar.y
    3|     test_files: input*.test,
                            ^ Unxpected token: '*', perhaps this is a glob, in which case it requires string quoting.
```



Diff:
diff --git a/cfgrammar/src/lib/header.rs b/cfgrammar/src/lib/header.rs
index c89f5c762..d591596be 100644
--- a/cfgrammar/src/lib/header.rs
+++ b/cfgrammar/src/lib/header.rs
@@ -3,7 +3,7 @@ use crate::{
     yacc::{
         parser::SpansKind, YaccGrammarError, YaccGrammarErrorKind, YaccKind, YaccOriginalActionKind,
     },
-    Location, Span,
+    Location, Span, Spanned,
 };
 use lazy_static::lazy_static;
 use regex::{Regex, RegexBuilder};
@@ -38,6 +38,15 @@ impl From<HeaderError<Span>> for YaccGrammarError {
     }
 }
 
+impl Spanned for HeaderError<Span> {
+    fn spans(&self) -> &[Span] {
+        self.locations.as_slice()
+    }
+    fn spanskind(&self) -> SpansKind {
+        self.spanskind()
+    }
+}
+
 // This is essentially a tuple that needs a newtype so we can implement `From` for it.
 // Thus we aren't worried about it being `pub`.
 #[derive(Debug, PartialEq)]
diff --git a/nimbleparse/src/diagnostics.rs b/nimbleparse/src/diagnostics.rs
index 0888b7b70..024934f95 100644
--- a/nimbleparse/src/diagnostics.rs
+++ b/nimbleparse/src/diagnostics.rs
@@ -1,4 +1,4 @@
-use std::{error::Error, fmt::Display, path::Path, str::FromStr};
+use std::{cell::OnceCell, error::Error, fmt::Display, path::Path};
 
 use cfgrammar::{
     newlinecache::NewlineCache,
@@ -16,7 +16,7 @@ use unicode_width::UnicodeWidthStr;
 pub struct SpannedDiagnosticFormatter<'a> {
     src: &'a str,
     path: &'a Path,
-    nlc: NewlineCache,
+    nlc: OnceCell<NewlineCache>,
 }
 
 fn pidx_prods_data<StorageT>(ast: &GrammarAST, pidx: PIdx<StorageT>) -> (Vec<String>, Vec<Span>)
@@ -36,11 +36,19 @@ where
 
 impl<'a> SpannedDiagnosticFormatter<'a> {
     #[allow(clippy::result_unit_err)]
-    pub fn new(src: &'a str, path: &'a Path) -> Result<Self, ()> {
-        Ok(Self {
+    pub fn new(src: &'a str, path: &'a Path) -> Self {
+        Self {
             src,
             path,
-            nlc: NewlineCache::from_str(src)?,
+            nlc: OnceCell::new(),
+        }
+    }
+
+    pub fn nlc(&self) -> &NewlineCache {
+        self.nlc.get_or_init(|| {
+            let mut nlc = NewlineCache::new();
+            nlc.feed(self.src);
+            nlc
         })
     }
 
@@ -60,7 +68,7 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
     pub fn file_location_msg(&self, msg: &str, span: Option<Span>) -> String {
         if let Some(span) = span {
             let (line, col) = self
-                .nlc
+                .nlc()
                 .byte_to_line_num_and_col_num(self.src, span.start())
                 .unwrap_or((0, 0));
             format!("{} at {}:{line}:{col}", msg, self.path.display())
@@ -85,11 +93,11 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
         underline_c: char,
     ) -> String {
         let mut out = String::new();
-        let (start_byte, end_byte) = self.nlc.span_line_bytes(span);
+        let (start_byte, end_byte) = self.nlc().span_line_bytes(span);
         // Produce an underline underneath a span which may cover multiple lines, and a message on the last line.
         let mut source_lines = self.src[start_byte..end_byte].lines().peekable();
         while let Some(source_line) = source_lines.next() {
-            let (line_start_byte, _) = self.nlc.span_line_bytes(span);
+            let (line_start_byte, _) = self.nlc().span_line_bytes(span);
             let span_offset_from_start = span.start() - line_start_byte;
 
             // An underline bounded by the current line.
@@ -99,7 +107,7 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
                     .min(span.start() + (source_line.len() - span_offset_from_start)),
             );
             let (line_num, _) = self
-                .nlc
+                .nlc()
                 .byte_to_line_num_and_col_num(self.src, span.start())
                 .expect("Span must correlate to a line in source");
             // Print the line_num/source text for the line.
@@ -137,13 +145,13 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
         let mut spans = e.spans().iter().enumerate().peekable();
         while let Some((span_num, span)) = spans.next() {
             let (line, _) = self
-                .nlc
+                .nlc()
                 .byte_to_line_num_and_col_num(self.src, span.start())
                 .unwrap_or((0, 0));
             let next_line = spans
                 .peek()
                 .map(|(_, span)| span)
-                .map(|s| self.nlc.byte_to_line_num(s.start()).unwrap_or(line))
+                .map(|s| self.nlc().byte_to_line_num(s.start()).unwrap_or(line))
                 .unwrap_or(line);
             // Is this line contiguous with the next, if not prefix it with dots.
             let dots = if next_line - line > 1 { "..." } else { "" };
@@ -180,7 +188,7 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
     ) -> String {
         let mut lines = spans
             .clone()
-            .map(|span| self.nlc.span_line_bytes(span))
+            .map(|span| self.nlc().span_line_bytes(span))
             .collect::<Vec<_>>();
         lines.dedup();
         assert!(lines.len() == 1);
@@ -192,10 +200,10 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
         //       ____      ___
         // indent    indent
         let (line_at_start, _) = self
-            .nlc
+            .nlc()
             .byte_to_line_num_and_col_num(self.src, first_span.start())
             .expect("Span should correlate to a line in source");
-        let (line_start_byte, end_byte) = self.nlc.span_line_bytes(*first_span);
+        let (line_start_byte, end_byte) = self.nlc().span_line_bytes(*first_span);
         // Print the line_num/source text for the line.
         out.push_str(&format!(
             "{}| {}\n",
@@ -304,7 +312,7 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
 
             let mut prod_lines = r_prod_spans
                 .iter()
-                .map(|span| self.nlc.span_line_bytes(*span))
+                .map(|span| self.nlc().span_line_bytes(*span))
                 .collect::<Vec<_>>();
             prod_lines.sort();
             prod_lines.dedup();
@@ -312,7 +320,7 @@ impl<'a> SpannedDiagnosticFormatter<'a> {
             for lines in &prod_lines {
                 let mut spans_on_line = Vec::new();
                 for span in &r_prod_spans {
-                    if lines == &self.nlc.span_line_bytes(*span) {
+                    if lines == &self.nlc().span_line_bytes(*span) {
                         spans_on_line.push(*span)
                     }
                 }
@@ -375,7 +383,7 @@ mod test {
     fn underline_multiline_span_test() {
         let s = "\naaaaaabbb\nbbb\nbbbb\n";
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(s, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(s, &test_path);
 
         let span = Span::new(7, 7 + 12);
         let out = format!(
@@ -413,7 +421,7 @@ mod test {
     fn underline_single_line_span_test() {
         let s = "\naaaaaabbb bbb bbbb\n";
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(s, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(s, &test_path);
 
         let span = Span::new(7, 7 + 12);
         let out = format!(
@@ -442,7 +450,7 @@ mod test {
     fn span_prefix() {
         let s = "\naaaaaabbb\nbbb\nbbbb\n";
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(s, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(s, &test_path);
         // For raw string alignment.
         let mut out = String::from("\n");
         // On occasion we want dots to signal that the lines are not contiguous.
@@ -473,7 +481,7 @@ mod test {
     fn span_prefix_2() {
         let s = "\n\n\n\n\n\n\n\n\n\n\naaaaaabbb\nbbb\nbbbb\n";
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(s, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(s, &test_path);
         let mut out = String::from("\n");
         // On occasion we want dots to signal that the lines are not contiguous.
         out.push_str(&formatter.prefixed_underline_span_with_text(
@@ -504,7 +512,7 @@ mod test {
         let crabs = " 🦀🦀🦀 ";
         let crustaceans = format!("\"{crabs}\n{crabs}\"");
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path);
         // For raw string alignment.
         let mut out = String::from("\n");
         out.push_str(&formatter.underline_span_with_text(
@@ -528,7 +536,7 @@ mod test {
         let lobster = "🦞";
         let crustaceans = format!("{crab}{lobster}{crab}{crab}{lobster}");
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path);
         // For raw string alignment.
         let mut out = String::from("\n");
         out.push_str(&formatter.prefixed_underline_span_with_text(
@@ -560,7 +568,7 @@ mod test {
     fn underline_single_line_spans_test() {
         let s = "\naaaaaabbb bbb bbbb\n";
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(s, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(s, &test_path);
         let spans = [(7, 10), (11, 14), (15, 19)]
             .iter()
             .map(|(i, j): &(usize, usize)| Span::new(*i, *j));
@@ -617,7 +625,7 @@ mod test {
         let lobster = "🦞";
         let crustaceans = format!("{crab}{lobster}{crab}{lobster}");
         let test_path = PathBuf::from("test");
-        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path).unwrap();
+        let formatter = SpannedDiagnosticFormatter::new(&crustaceans, &test_path);
         // For raw string alignment.
         let mut out = String::from("\n");
         let spans = [
diff --git a/nimbleparse/src/main.rs b/nimbleparse/src/main.rs
index 3e4cd3410..17ddc5581 100644
--- a/nimbleparse/src/main.rs
+++ b/nimbleparse/src/main.rs
@@ -1,7 +1,7 @@
 mod diagnostics;
 use crate::diagnostics::*;
 use cfgrammar::{
-    header::{GrmtoolsSectionParser, Header, HeaderValue, Value},
+    header::{GrmtoolsSectionParser, Header, HeaderError, HeaderValue, Value},
     markmap::Entry,
     yacc::{ast::ASTWithValidityInfo, YaccGrammar, YaccKind, YaccOriginalActionKind},
     Location, Span,
@@ -141,12 +141,12 @@ fn main() {
     };
     let entry = match header.entry("yacckind".to_string()) {
         Entry::Occupied(_) => unreachable!("Header should be empty"),
-        Entry::Vacant(v) => v.occupied_entry(),
+        Entry::Vacant(v) => v,
     };
     match matches.opt_str("y") {
         None => {}
         Some(s) => {
-            entry.insert(HeaderValue(
+            entry.insert_entry(HeaderValue(
                 Location::CommandLine,
                 Value::try_from(match &*s.to_lowercase() {
                     "eco" => YaccKind::Eco,
@@ -165,13 +165,13 @@ fn main() {
 
     let lex_l_path = PathBuf::from(&matches.free[0]);
     let lex_src = read_file(&lex_l_path);
+    let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, &lex_l_path);
     let mut lexerdef = match LRNonStreamingLexerDef::<DefaultLexerTypes<u32>>::from_str(&lex_src) {
         Ok(ast) => ast,
         Err(errs) => {
-            let formatter = SpannedDiagnosticFormatter::new(&lex_src, &lex_l_path).unwrap();
-            eprintln!("{ERROR}{}", formatter.file_location_msg("", None));
+            eprintln!("{ERROR}{}", lex_diag.file_location_msg("", None));
             for e in errs {
-                eprintln!("{}", indent(&formatter.format_error(e).to_string(), "    "));
+                eprintln!("{}", indent(&lex_diag.format_error(e).to_string(), "    "));
             }
             process::exit(1);
         }
@@ -179,6 +179,7 @@ fn main() {
 
     let yacc_y_path = PathBuf::from(&matches.free[1]);
     let yacc_src = read_file(&yacc_y_path);
+    let yacc_diag = SpannedDiagnosticFormatter::new(&yacc_src, &yacc_y_path);
     let yk_val = header.get("yacckind");
     if yk_val.is_none() {
         let parsed_header = GrmtoolsSectionParser::new(&yacc_src, true).parse();
@@ -190,12 +191,12 @@ fn main() {
             }
             Err(errs) => {
                 eprintln!(
-                    "Error(s) parsing `%grmtools` section:\n {}\n",
-                    errs.iter()
-                        .map(|e| e.to_string())
-                        .collect::<Vec<_>>()
-                        .join("\n")
+                    "{ERROR}{}",
+                    yacc_diag.file_location_msg(" parsing the `%grmtools` section:", None)
                 );
+                for e in errs {
+                    eprintln!("{}", indent(&yacc_diag.format_error(e).to_string(), "    "));
+                }
                 std::process::exit(1);
             }
         }
@@ -211,7 +212,25 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                eprintln!(
+                    "{ERROR}{}",
+                    yacc_diag.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                let spanned_e: HeaderError<Span> = HeaderError {
+                    kind: e.kind,
+                    locations: e
+                        .locations
+                        .iter()
+                        .map(|l| match l {
+                            Location::Span(span) => *span,
+                            _ => unreachable!("All reachable errors should contain spans"),
+                        })
+                        .collect::<Vec<_>>(),
+                };
+                eprintln!(
+                    "{}",
+                    indent(&yacc_diag.format_error(spanned_e).to_string(), "    ")
+                );
                 process::exit(1)
             }
             Ok(rk) => rk,
@@ -222,28 +241,24 @@ fn main() {
     };
     let warnings = ast_validation.ast().warnings();
     let res = YaccGrammar::new_from_ast_with_validity_info(&ast_validation);
-    let mut yacc_diagnostic_formatter: Option<SpannedDiagnosticFormatter> = None;
     let grm = match res {
         Ok(x) => {
             if !warnings.is_empty() {
-                let formatter = SpannedDiagnosticFormatter::new(&yacc_src, &yacc_y_path).unwrap();
-                eprintln!("{WARNING}{}", formatter.file_location_msg("", None));
+                eprintln!("{WARNING}{}", yacc_diag.file_location_msg("", None));
                 for w in warnings {
-                    eprintln!("{}", indent(&formatter.format_warning(w), "    "));
+                    eprintln!("{}", indent(&yacc_diag.format_warning(w), "    "));
                 }
-                yacc_diagnostic_formatter = Some(formatter);
             }
             x
         }
         Err(errs) => {
-            let formatter = SpannedDiagnosticFormatter::new(&yacc_src, &yacc_y_path).unwrap();
-            eprintln!("{ERROR}{}", formatter.file_location_msg("", None));
+            eprintln!("{ERROR}{}", yacc_diag.file_location_msg("", None));
             for e in errs {
-                eprintln!("{}", indent(&e.to_string(), "    "));
+                eprintln!("{}", indent(&yacc_diag.format_error(e).to_string(), "    "));
             }
-            eprintln!("{WARNING}{}", formatter.file_location_msg("", None));
+            eprintln!("{WARNING}{}", yacc_diag.file_location_msg("", None));
             for w in warnings {
-                eprintln!("{}", indent(&formatter.format_warning(w), "    "));
+                eprintln!("{}", indent(&yacc_diag.format_warning(w), "    "));
             }
             process::exit(1);
         }
@@ -262,14 +277,6 @@ fn main() {
 
     if !quiet {
         if let Some(c) = stable.conflicts() {
-            let formatter = if let Some(yacc_diagnostic_formatter) = &yacc_diagnostic_formatter {
-                yacc_diagnostic_formatter
-            } else {
-                let formatter = SpannedDiagnosticFormatter::new(&yacc_src, &yacc_y_path).unwrap();
-                yacc_diagnostic_formatter = Some(formatter);
-                yacc_diagnostic_formatter.as_ref().unwrap()
-            };
-
             let pp_rr = if let Some(i) = grm.expectrr() {
                 i != c.rr_len()
             } else {
@@ -289,7 +296,7 @@ fn main() {
             if (pp_rr || pp_sr) && !dump_state_graph {
                 println!("Stategraph:\n{}\n", sgraph.pp_core_states(&grm));
             }
-            formatter.handle_conflicts::<DefaultLexerTypes<u32>>(
+            yacc_diag.handle_conflicts::<DefaultLexerTypes<u32>>(
                 &grm,
                 ast_validation.ast(),
                 c,
@@ -303,19 +310,18 @@ fn main() {
     {
         if !quiet {
             if let Some(token_spans) = missing_from_lexer {
-                let formatter = SpannedDiagnosticFormatter::new(&yacc_src, &yacc_y_path).unwrap();
                 let warn_indent = " ".repeat(WARNING.len());
                 eprintln!(
                     "{WARNING} these tokens are not referenced in the lexer but defined as follows"
                 );
                 eprintln!(
                     "{warn_indent} {}",
-                    formatter.file_location_msg("in the grammar", None)
+                    yacc_diag.file_location_msg("in the grammar", None)
                 );
                 for span in token_spans {
                     eprintln!(
                         "{}",
-                        formatter.underline_span_with_text(
+                        yacc_diag.underline_span_with_text(
                             span,
                             "Missing from lexer".to_string(),
                             '^'
@@ -325,19 +331,18 @@ fn main() {
                 eprintln!();
             }
             if let Some(token_spans) = missing_from_parser {
-                let formatter = SpannedDiagnosticFormatter::new(&lex_src, &lex_l_path).unwrap();
                 let err_indent = " ".repeat(ERROR.len());
                 eprintln!(
                     "{ERROR} these tokens are not referenced in the grammar but defined as follows"
                 );
                 eprintln!(
                     "{err_indent} {}",
-                    formatter.file_location_msg("in the lexer", None)
+                    lex_diag.file_location_msg("in the lexer", None)
                 );
                 for span in token_spans {
                     eprintln!(
                         "{}",
-                        formatter.underline_span_with_text(
+                        lex_diag.underline_span_with_text(
                             span,
                             "Missing from parser".to_string(),
                             '^'


Reviews:
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T07:07:00Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T07:31:27Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T07:35:13Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T08:27:06Z
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T09:33:57Z
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T09:34:04Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T16:03:32Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-12T16:10:28Z
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-12T16:14:34Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-15T01:41:49Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-15T01:44:39Z
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-15T07:03:18Z
user: login: ratmice
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-16T06:24:18Z
user: login: ltratt
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-16T09:10:45Z

Review comments:
diff_hunk: @@ -211,7 +220,33 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
+                eprintln!(
+                    "{ERROR}{}",
+                    formatter.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                // I believe that this can only ever be true, but I haven't tried to prove it nor do the types ensure it.
path: nimbleparse/src/main.rs
user: login: ltratt
body: If we're fairly confident of this, I would make this an `unwrap` (or similar). If we are wrong, we'll find out, and can then readd the complexity back in.
created_at: 2025-05-12T07:04:20Z
author_association: MEMBER
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ltratt
body: I'm not totally sure I've understood why this needs to be a `OnceCell`. I may have misunderstood the control flow, but I *think* the `SpannedDiagnosticFormatter` is initialised on two separate codepaths: couldn't we just use a normal `let yacc_diag = SpannedDiagnosticFormater::...` on each of those paths? Feel free to tell me I've missed out on something!
created_at: 2025-05-12T07:06:58Z
author_association: MEMBER
diff_hunk: @@ -325,7 +353,8 @@ fn main() {
                 eprintln!();
             }
             if let Some(token_spans) = missing_from_parser {
-                let formatter = SpannedDiagnosticFormatter::new(&lex_src, &lex_l_path).unwrap();
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
path: nimbleparse/src/main.rs
user: login: ratmice
body: This one is using `yacc_src` where it should be using `lex_src`, definitely needs to change.
created_at: 2025-05-12T07:31:27Z
author_association: COLLABORATOR
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ratmice
body: The main reason to build it lazily and once is that it involves building a `NewLineCache` and a pass over the entire source building the data structure containing the offsets of each line.  So the current code (and the usage of once cell in this patch) is trying to do two things: 

1. Avoid building a `NewlineCache` entirely unless an error actually occurs.
2. Avoid rebuilding a `NewlineCache` when one was already built for the same source text.

I believe the only case where second thing happens *across branches* is when a *warning* happens and then, 
in a later branch a conflict occurs and gets printed.  We could also perhaps relax the error handling to also print conflict errors.

There is also the `missing_from_lexer` and `missing_from_parser`, we could non-fatal errors in header parsing,
like we might want to complain about `%grmtools{ some_random_unknown_header_value }`, (I think we do in both `CTParserBuilder` and `CTLexerBuilder` but I may have forgotten that in nimbleparse.



created_at: 2025-05-12T07:35:12Z
author_association: COLLABORATOR
diff_hunk: @@ -211,7 +220,33 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
+                eprintln!(
+                    "{ERROR}{}",
+                    formatter.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                // I believe that this can only ever be true, but I haven't tried to prove it nor do the types ensure it.
path: nimbleparse/src/main.rs
user: login: ratmice
body: We can't exactly use `unwrap`, but we can just pull this out of the `if` and remove the `else`, I guess that is a worthwhile cleanup

```
                  let spanned_e: HeaderError<Span> = HeaderError {
                      kind: e.kind,
                      locations: e
                          .locations
                          .iter()
                          .map(|l| match l {
                              Location::Span(span) => *span,
                              _ => unreachable!(),
                          })
                          .collect::<Vec<_>>(),
                  };
                  eprintln!(
                      "{}",
                      indent(&formatter.format_error(spanned_e).to_string(), "    ")
                  );
```

The reason I believe that this case cannot occur, because: 

1. Values passed during argument parsing are pre-checked for validity and would have already produced errors.
2. Other values lacking spans that derive from a e.g `From<RecoveryKind> for Value<Location>` impl, cannot produce errors during the corresponding inverse `TryFrom<Value<Location>> for RecoveryKind`
3. All other values have corresponding spans.
4. The merge process being used isn't mutually exclusive so can't introduce errors between otherwise valid values.

(I want to stress that those 4 points are just the hypothesis behind why I believe these are unreachable)

Making thats `unreachable!("args have already been checked, values derived from From impls are infallible, and all other values should have spans");`

The first 3 of those are straight-forward it's cases like that last one though where this line of thinking becomes somewhat tenuous, and give me pause mainly due to questioning the fragility and whether that will always be the case.
created_at: 2025-05-12T08:27:06Z
author_association: COLLABORATOR
diff_hunk: @@ -211,7 +220,33 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
+                eprintln!(
+                    "{ERROR}{}",
+                    formatter.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                // I believe that this can only ever be true, but I haven't tried to prove it nor do the types ensure it.
path: nimbleparse/src/main.rs
user: login: ltratt
body: I tend to agree that these are probably `unreachable`.
created_at: 2025-05-12T09:33:57Z
author_association: MEMBER
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ltratt
body: Right, got it.
created_at: 2025-05-12T09:34:03Z
author_association: MEMBER
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ratmice
body: One thing we could try is moving to `OnceCell<NewlineCache>` inside `SpannedDiagnosticFormatter` then initializing it in an accessor method `self.nlc()`, that also allows us to remove the `unwrap` from the lazy initialization code I guess.
created_at: 2025-05-12T16:03:32Z
author_association: COLLABORATOR
diff_hunk: @@ -211,7 +220,33 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
+                eprintln!(
+                    "{ERROR}{}",
+                    formatter.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                // I believe that this can only ever be true, but I haven't tried to prove it nor do the types ensure it.
path: nimbleparse/src/main.rs
user: login: ratmice
body: It'll be a few days before I'm able to make/test this change, will try the above change out then.
created_at: 2025-05-12T16:10:28Z
author_association: COLLABORATOR
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ltratt
body: If that makes it clearer, go for it, but if it makes it worse, let's keep what we've got.
created_at: 2025-05-12T16:14:34Z
author_association: MEMBER
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ratmice
body: I did that in [fb27a50](https://github.com/softdevteam/grmtools/pull/564/commits/fb27a5008b306c02f4f3859e364d4229857f020d), and I felt it improved things, curious what you think?
created_at: 2025-05-15T01:41:49Z
author_association: COLLABORATOR
diff_hunk: @@ -211,7 +220,33 @@ fn main() {
     let recoverykind = if let Some(HeaderValue(_, rk_val)) = header.get("recoverer") {
         match RecoveryKind::try_from(rk_val) {
             Err(e) => {
-                eprintln!("{e}");
+                let formatter =
+                    yacc_diag.get_or_init(|| spanned_formatter(&yacc_src, &yacc_y_path));
+                eprintln!(
+                    "{ERROR}{}",
+                    formatter.file_location_msg(" parsing the `%grmtools` section:", None)
+                );
+                // I believe that this can only ever be true, but I haven't tried to prove it nor do the types ensure it.
path: nimbleparse/src/main.rs
user: login: ratmice
body: I ran across an unrelated issue while testing this, fixed in [8c626cc](https://github.com/softdevteam/grmtools/pull/564/commits/8c626ccb0d74ab504106bfe57655468dfff90fab)
But otherwise did this change in [11648c6](https://github.com/softdevteam/grmtools/pull/564/commits/11648c67cadb29e7b14147be30fcc55663accc4a)
created_at: 2025-05-15T01:44:39Z
author_association: COLLABORATOR
diff_hunk: @@ -86,7 +87,13 @@ where
     (missing_from_lexer, missing_from_parser)
 }
 
+fn spanned_formatter<'a>(input: &'a str, path: &'a Path) -> SpannedDiagnosticFormatter<'a> {
+    SpannedDiagnosticFormatter::new(input, path).unwrap()
+}
+
 fn main() {
+    let yacc_diag: OnceCell<SpannedDiagnosticFormatter> = OnceCell::new();
path: nimbleparse/src/main.rs
user: login: ltratt
body: Works for me!
created_at: 2025-05-15T07:03:18Z
author_association: MEMBER
diff_hunk: @@ -141,12 +141,12 @@ fn main() {
     };
     let entry = match header.entry("yacckind".to_string()) {
         Entry::Occupied(_) => unreachable!("Header should be empty"),
-        Entry::Vacant(v) => v.occupied_entry(),
+        Entry::Vacant(v) => v,
path: nimbleparse/src/main.rs
user: login: ratmice
body: I think rather than what I discussed in the commit message, of adding a different `key_entry`, it's best to just implement the `mark_required` and `set_merge_behavior` functions for `VacantEntry` directly, perhaps `mark_used` too but I can't think of a reason to mark a vacant entry used.

I still think it's best to do as a separate patch, but let me know if you feel otherwise.
created_at: 2025-05-16T06:24:18Z
author_association: COLLABORATOR
diff_hunk: @@ -141,12 +141,12 @@ fn main() {
     };
     let entry = match header.entry("yacckind".to_string()) {
         Entry::Occupied(_) => unreachable!("Header should be empty"),
-        Entry::Vacant(v) => v.occupied_entry(),
+        Entry::Vacant(v) => v,
path: nimbleparse/src/main.rs
user: login: ltratt
body: Doing it in another PR is fine!
created_at: 2025-05-16T09:10:44Z
author_association: MEMBER

PR comments:
user: login: ratmice
created_at: 2025-05-11T22:54:02Z
body: One thing I notice in the output is that the first instance doesn't have a `^` caret and I'm not certain why, but it will be a few days before I can investigate it.

I am also wondering whether you think it is worth trying larger follow up where we try moving this diagnostics formatting code to a `support` library (or just making it a hidden module in cfgrammar), and use them from within `CTParserBuilder`, `CTLexerBuilder` and `nimbleparse`, I believe all my other failed attempts have been to just make API so people can use whatever error printing library they want within their `build.rs`.

This would be a different approach where since it doesn't try to add any API where people can customize error printing, but just shares the nimbleparse error printing code between the aformentioned things?
author_association: COLLABORATOR
user: login: ltratt
created_at: 2025-05-12T07:08:03Z
body: > I am also wondering whether you think it is worth trying larger follow up where we try moving this diagnostics formatting code to a support library (or just making it a hidden module in cfgrammar)

My gut reaction is to first see what we can break into a separate module *within* nimbleparse; and, depending on how that goes, maybe we can then put that in another crate. Warning: this might not be a good idea on my part!
author_association: MEMBER
user: login: ratmice
created_at: 2025-05-12T07:52:21Z
body: > My gut reaction is to first see what we can break into a separate module _within_ nimbleparse; and, depending on how that goes, maybe we can then put that in another crate. Warning: this might not be a good idea on my part!

The `diagnostics.rs` stuff is basically already is it's own module, but the problem/need to move it outside nimbleparse is more so that it can be used within `CTParserBuilder` and `pub` so that it can be used within `CTLexerBuilder` that was the motiviation behind proposing that it be `#[doc(hidden)]`.

It seemed like something I haven't tried yet at least
author_association: COLLABORATOR
user: login: ltratt
created_at: 2025-05-15T07:04:14Z
body: Please squash.
author_association: MEMBER
user: login: ratmice
created_at: 2025-05-15T23:20:21Z
body: Squashed, but I left a long commit message for [be35407](https://github.com/softdevteam/grmtools/pull/564/commits/be354073054128046e8bea7d80a2420f0282aa67) describing that the API is somewhat prone to misuse and perhaps some future changes that could be made to avoid this kind of error in the future.

I'm not sure it is imperative that we fix that immediately, considering it's hidden API and the only other usage of the API looks fine.
author_association: COLLABORATOR