Repository: pest-parser/pest (language: rust)

Issue:
Tags on rules that can yield more than one pair
### Discussed in https://github.com/pest-parser/pest/discussions/979

<div type='discussions-op-text'>

<sup>Originally posted by **agatti** February 15, 2024</sup>
Hi there!

I'm using pest for the very first time and I'm not sure if I bumped into a bug or not.  So, given this grammar fragment on the latest stable version (2.7.7 with "grammar-extras"):

```pest
tag = {
	// stuff here
}

container = {
	// stuff here
	"{" ~ #tags=tag* ~ "}"
	// more stuff here
}
```

I'd assume that calling `find_tagged("tags")` on a `Pair<Rule::container>` would return all the `Pair<Rule::tag>` that are between brackets.  However what I see is that `find_tagged` returns only the _last_ tag seen by the parser, as if there's just one available pair per tag and it gets overwritten as the parser goes through data.  The grammar seems to be correct as if I remove the `#tags=` bit and do the inner pair iteration myself all the tags are there.

I've read the pest book and I couldn't find anything saying there are limitations on tags attached to rules that may yield more than a pair.  Is this a bug, an undocumented limitation, or am I doing things wrong somewhere?</div>


Discussion / hints:

Fix patch:
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
index 4947d191..f22d7173 100644
--- a/debugger/Cargo.toml
+++ b/debugger/Cargo.toml
@@ -1,9 +1,12 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
-authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
+authors = [
+    "Dragoș Tiselice <dragostiselice@gmail.com>",
+    "Tomas Tauber <me@tomtau.be>",
+]
 homepage = "https://pest.rs/"
 repository = "https://github.com/pest-parser/pest"
 documentation = "https://docs.rs/pest"
@@ -14,10 +17,14 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_meta = { path = "../meta", version = "2.7.7" }
-pest_vm = { path = "../vm", version = "2.7.7" }
-reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
+pest = { path = "../pest", version = "2.7.8" }
+pest_meta = { path = "../meta", version = "2.7.8" }
+pest_vm = { path = "../vm", version = "2.7.8" }
+reqwest = { version = "= 0.11.13", default-features = false, features = [
+    "blocking",
+    "json",
+    "default-tls",
+] }
 rustyline = "10"
 serde_json = "1"
 thiserror = "1"
diff --git a/derive/Cargo.toml b/derive/Cargo.toml
index eecdfde8..58d56c58 100644
--- a/derive/Cargo.toml
+++ b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.7.7", default-features = false }
-pest_generator = { path = "../generator", version = "2.7.7", default-features = false }
+pest = { path = "../pest", version = "2.7.8", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.8", default-features = false }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
index 166da150..eb80f92f 100644
--- a/generator/Cargo.toml
+++ b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -22,8 +22,8 @@ grammar-extras = ["pest_meta/grammar-extras"]
 export-internal = []
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7", default-features = false }
-pest_meta = { path = "../meta", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.8" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
diff --git a/generator/src/generator.rs b/generator/src/generator.rs
index f5fa5a1f..042b2ff9 100644
--- a/generator/src/generator.rs
+++ b/generator/src/generator.rs
@@ -571,12 +571,42 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
             }
         }
         #[cfg(feature = "grammar-extras")]
-        OptimizedExpr::NodeTag(expr, tag) => {
-            let expr = generate_expr(*expr);
-            quote! {
-                #expr.and_then(|state| state.tag_node(#tag))
+        OptimizedExpr::NodeTag(expr, tag) => match *expr {
+            OptimizedExpr::Opt(expr) => {
+                let expr = generate_expr(*expr);
+                quote! {
+                    state.optional(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
             }
-        }
+            OptimizedExpr::Rep(expr) => {
+                let expr = generate_expr(*expr);
+                quote! {
+                    state.sequence(|state| {
+                        state.optional(|state| {
+                            #expr.and_then(|state| {
+                                state.repeat(|state| {
+                                    state.sequence(|state| {
+                                        super::hidden::skip(
+                                            state
+                                        ).and_then(|state| {
+                                            #expr.and_then(|state| state.tag_node(#tag))
+                                        })
+                                    })
+                                })
+                            }).and_then(|state| state.tag_node(#tag))
+                        })
+                    })
+                }
+            }
+            expr => {
+                let expr = generate_expr(expr);
+                quote! {
+                    #expr.and_then(|state| state.tag_node(#tag))
+                }
+            }
+        },
     }
 }
 
@@ -726,12 +756,32 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
             }
         }
         #[cfg(feature = "grammar-extras")]
-        OptimizedExpr::NodeTag(expr, tag) => {
-            let expr = generate_expr_atomic(*expr);
-            quote! {
-                #expr.and_then(|state| state.tag_node(#tag))
+        OptimizedExpr::NodeTag(expr, tag) => match *expr {
+            OptimizedExpr::Opt(expr) => {
+                let expr = generate_expr_atomic(*expr);
+
+                quote! {
+                    state.optional(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
             }
-        }
+            OptimizedExpr::Rep(expr) => {
+                let expr = generate_expr_atomic(*expr);
+
+                quote! {
+                    state.repeat(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
+            }
+            expr => {
+                let expr = generate_expr_atomic(expr);
+                quote! {
+                    #expr.and_then(|state| state.tag_node(#tag))
+                }
+            }
+        },
     }
 }
 
diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
index 4f3f2df4..4f1a4e0a 100644
--- a/grammars/Cargo.toml
+++ b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_derive = { path = "../derive", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
+pest_derive = { path = "../derive", version = "2.7.8" }
 
 [dev-dependencies]
 criterion = "0.5"
diff --git a/meta/Cargo.toml b/meta/Cargo.toml
index 920e6df2..71fdbd9a 100644
--- a/meta/Cargo.toml
+++ b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -12,11 +12,17 @@ categories = ["parsing"]
 license = "MIT OR Apache-2.0"
 readme = "_README.md"
 exclude = ["src/grammar.pest"]
-include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"]
+include = [
+    "Cargo.toml",
+    "src/**/*",
+    "src/grammar.rs",
+    "_README.md",
+    "LICENSE-*",
+]
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
 once_cell = "1.8.0"
 
 [build-dependencies]
diff --git a/pest/Cargo.toml b/pest/Cargo.toml
index a0f4b7df..3ffb3ebf 100644
--- a/pest/Cargo.toml
+++ b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
index ba38f140..fc597f1f 100644
--- a/vm/Cargo.toml
+++ b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_meta = { path = "../meta", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
+pest_meta = { path = "../meta", version = "2.7.8" }
 
 [features]
 grammar-extras = ["pest_meta/grammar-extras"]


Test patch:
diff --git a/derive/tests/implicit.rs b/derive/tests/implicit.rs
index 48c88021..20dda844 100644
--- a/derive/tests/implicit.rs
+++ b/derive/tests/implicit.rs
@@ -35,3 +35,12 @@ fn test_implicit_whitespace() {
     assert_eq!(pairs.clone().find_tagged("one_comp").count(), 2);
     assert_eq!(pairs.find_tagged("one_array").count(), 2);
 }
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+fn test_implicit_whitespace_multitag() {
+    let successful_parse = TestImplicitParser::parse(Rule::program, "a a a");
+    assert!(successful_parse.is_ok());
+    let pairs = successful_parse.unwrap();
+    assert_eq!(pairs.clone().find_tagged("tail").count(), 2);
+}
diff --git a/derive/tests/opt.pest b/derive/tests/opt.pest
new file mode 100644
index 00000000..b5ff4661
--- /dev/null
+++ b/derive/tests/opt.pest
@@ -0,0 +1,8 @@
+expr = {
+  SOI ~
+  #prefix=(STAR)? ~ #suffix=DOT?
+  ~ EOI
+}
+
+STAR={"*"}
+DOT={"."}
diff --git a/derive/tests/opt.rs b/derive/tests/opt.rs
new file mode 100644
index 00000000..5c07f84d
--- /dev/null
+++ b/derive/tests/opt.rs
@@ -0,0 +1,42 @@
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
+// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. All files in the project carrying such notice may not be copied,
+// modified, or distributed except according to those terms.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+extern crate pest;
+extern crate pest_derive;
+
+#[cfg(feature = "grammar-extras")]
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/opt.pest"]
+struct TestOptParser;
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+fn test_opt_tag() {
+    let successful_parse = TestOptParser::parse(Rule::expr, "*");
+    assert!(successful_parse.is_ok());
+    let pairs = successful_parse.unwrap();
+    assert!(pairs.find_first_tagged("prefix").is_some());
+    assert!(pairs.find_first_tagged("suffix").is_none());
+
+    // Test with no STAR or DOT
+    let parse_no_components = TestOptParser::parse(Rule::expr, "");
+    assert!(parse_no_components.is_ok());
+    let pairs_no_components = parse_no_components.unwrap();
+    assert!(pairs_no_components.find_first_tagged("prefix").is_none());
+    assert!(pairs_no_components.find_first_tagged("suffix").is_none());
+
+    // Test with only DOT
+    let parse_only_dot = TestOptParser::parse(Rule::expr, ".");
+    assert!(parse_only_dot.is_ok());
+    let pairs_only_dot = parse_only_dot.unwrap();
+    assert!(pairs_only_dot.find_first_tagged("prefix").is_none());
+    assert!(pairs_only_dot.find_first_tagged("suffix").is_some());
+}