Repository: lalrpop/lalrpop (language: rust)

Issue:
Allow multiline comments in grammar files
Is there any particular reason why `lalrpop` doesn't support multiline comments (`/* ... */`)? I just noticed it when trying to put a copy of my reference grammar in my `*.lalrpop` file.

For Example:

```rust
use ast::{Atom, BinOp, BinaryOp, Expr};

/*
Original Grammar
================

Expressions
-----------

Expression         ::= XOR_Expression {'OR' XOR_Expression}
XOR_Expression     ::= AND_Expression {'XOR' AND_Expression}
   ...
ST_FunctionInput   ::= [VariableName ':='] Expression
*/

grammar<'input>;

...
```


Discussion / hints:

Fix patch:
diff --git a/lalrpop/src/build/mod.rs b/lalrpop/src/build/mod.rs
index 250c23ac6..5420e6dde 100644
--- a/lalrpop/src/build/mod.rs
+++ b/lalrpop/src/build/mod.rs
@@ -305,6 +305,9 @@ fn parse_and_normalize_grammar(session: &Session, file_text: &FileText) -> io::R
                 tok::ErrorCode::UnterminatedCode => {
                     "unterminated code block; perhaps a missing `;`, `)`, `]` or `}`?"
                 }
+                tok::ErrorCode::UnterminatedBlockComment => {
+                    "unterminated block comment; missing `*/`?"
+                }
             };
 
             report_error(
diff --git a/lalrpop/src/tok/mod.rs b/lalrpop/src/tok/mod.rs
index 363834a7c..2b5afafed 100644
--- a/lalrpop/src/tok/mod.rs
+++ b/lalrpop/src/tok/mod.rs
@@ -26,6 +26,7 @@ pub enum ErrorCode {
     UnterminatedAttribute,
     UnterminatedCode,
     ExpectedStringLiteral,
+    UnterminatedBlockComment,
 }
 
 fn error<T>(c: ErrorCode, l: usize) -> Result<T, Error> {
@@ -353,6 +354,13 @@ impl<'input> Tokenizer<'input> {
                         self.take_until(|c| c == '\n');
                         continue;
                     }
+                    Some((_, '*')) => {
+                        self.bump(); // Skip over the *
+                        match self.block_comment(idx0) {
+                            Err(err) => Some(Err(err)),
+                            _ => continue,
+                        }
+                    }
                     _ => Some(error(UnrecognizedToken, idx0)),
                 },
                 Some((idx0, '-')) => match self.bump() {
@@ -459,8 +467,15 @@ impl<'input> Tokenizer<'input> {
                     continue;
                 } else if c == '/' {
                     self.bump();
-                    if let Some((_, '/')) = self.lookahead {
-                        self.take_until(|c| c == '\n');
+                    match self.lookahead {
+                        Some((_, '/')) => {
+                            self.take_until(|c| c == '\n');
+                        }
+                        Some((_, '*')) => {
+                            self.bump(); // Skip over the *
+                            self.block_comment(idx)?
+                        }
+                        _ => {}
                     }
                     continue;
                 } else if open_delims.find(c).is_some() {
@@ -564,6 +579,49 @@ impl<'input> Tokenizer<'input> {
         }
     }
 
+    fn block_comment(&mut self, idx0: usize) -> Result<(), Error> {
+        #[derive(PartialEq, Eq, Copy, Clone)]
+        enum State {
+            Initial,
+            Slash,
+            Star,
+            Complete,
+        }
+
+        let mut depth = 1;
+        let mut state = State::Initial;
+
+        let end_of_comment = |c: char| {
+            state = match (state, c) {
+                (State::Initial | State::Star, '*') => State::Star,
+                (State::Initial | State::Slash, '/') => State::Slash,
+                (State::Slash, '*') => {
+                    depth += 1;
+                    State::Initial
+                }
+                (State::Star, '/') => {
+                    depth -= 1;
+                    if depth == 0 {
+                        State::Complete
+                    } else {
+                        State::Initial
+                    }
+                }
+                _ => State::Initial,
+            };
+
+            state == State::Complete
+        };
+
+        match self.take_until(end_of_comment) {
+            Some(_) => {
+                self.bump();
+                Ok(())
+            }
+            None => error(UnterminatedBlockComment, idx0),
+        }
+    }
+
     // parses `r#"..."#` (for some number of #), starts after the `r`
     // has been consumed; idx0 points at the `r`
     fn regex_literal(&mut self, idx0: usize) -> Result<Spanned<Tok<'input>>, Error> {


Test patch:
diff --git a/lalrpop/src/tok/test.rs b/lalrpop/src/tok/test.rs
index febeba5da..2c2663e74 100644
--- a/lalrpop/src/tok/test.rs
+++ b/lalrpop/src/tok/test.rs
@@ -69,6 +69,114 @@ fn eol_comment() {
     );
 }
 
+#[test]
+fn block_comment() {
+    test(
+        "extern /* This is a block comment */$ foo",
+        vec![
+            ("~~~~~~                                   ", Extern),
+            ("                                      ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_in_code() {
+    test(
+        "=> ( test /* foo ) */ ),",
+        vec![
+            (
+                "~~~~~~~~~~~~~~~~~~~~~~~ ",
+                EqualsGreaterThanCode(" ( test /* foo ) */ )"),
+            ),
+            ("                       ~", Comma),
+        ],
+    );
+}
+
+#[test]
+fn nested_block_comment() {
+    test(
+        "extern /* This is a /* nested */ block comment */$ foo",
+        vec![
+            (
+                "~~~~~~                                                ",
+                Extern,
+            ),
+            (
+                "                                                   ~~~",
+                Id("foo"),
+            ),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_3_star() {
+    test(
+        "extern /***/$ foo",
+        vec![
+            ("~~~~~~           ", Extern),
+            ("              ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_nested_3_star_with_linefeeds() {
+    test(
+        "extern /** /***/ $*/$ foo",
+        vec![
+            ("~~~~~~                   ", Extern),
+            ("                      ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_5_star() {
+    test(
+        "extern /*****/$ foo",
+        vec![
+            ("~~~~~~             ", Extern),
+            ("                ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_1_2_star() {
+    test(
+        "extern /* **/$ foo",
+        vec![
+            ("~~~~~~            ", Extern),
+            ("               ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn block_comment_extra_slashes() {
+    test(
+        "extern /*//**/*/$ foo",
+        vec![
+            ("~~~~~~               ", Extern),
+            ("                  ~~~", Id("foo")),
+        ],
+    );
+}
+
+#[test]
+fn unterminated_block_comment() {
+    test_err(
+        "/* This is unterminated",
+        (
+            "~                      ",
+            ErrorCode::UnterminatedBlockComment,
+        ),
+    )
+}
+
 #[test]
 fn code1() {
     test(