Repository: pest-parser/pest (language: rust)

Issue:
`one or more` rules consume trailing whitespace when exactly matched once
Given the following grammar: 

```
a = { "a" ~ b+ | a }
b = { "b" }
root = { SOI ~ a* ~ EOI }
```

Then, parsing the following text: 
```
a b

```
Will associate the trailing newline after `b` with the rule for `a`, instead of the the rule for `root`.

That seems to work properly with `zero-or-more` expressions, where the whitespace will be associated with the `root` rule. 

Also, it works properly for more than one `b`. The following example:
```
a b b

```
Puts the whitespace correctly inside root. 


Discussion / hints:
After looking through the code, I'd say this is because `Expr::RepOnce` is translated to a sequence. 

E.g. for pest, `b+` is equal to `b ~ b*`.
Proposed fix in #397.
Ah, finally. I knew this day was going to come. Unfortunately, this implementation bug is probably something that people make use of in their logic. Fixing this might actually break them, but I'm happy to give it a try regardless maybe there's no actual dependee that uses it.

I have a fix for 3.0, but I haven't had that much time to work on it.
This just bit us. Are there still plans to resolve this? Is there evidence that people rely on this bug?

And lastly, is there a good workaround? The obvious idea would be to strip the white space off of everything pest returns, but I suspect there is a more elegant way, since otherwise there probably would be more people commenting here.

In our case we are having trouble to parse data in the format "x <- foo" (see [this page]).

Also, otherwise we are having a great time using pest, thank you!

cc @siccegge @chrisbrzuska

**Update**:

I found a satisfying workaround, documented here for future visitors. Instead of
```pest
identifier = { (ASCII_ALPHA | "_")+ }
```
I now use
```pest
identifier = @{ (ASCII_ALPHA | "_") ~ (ASCII_ALPHA | "_")* }
```
where `@` means that no internal white space is allowed and all subrules are muted (which is not important here since there are no internal rules).

[this page]: https://codimd.faui2k.cloud/s/kjeothgO_#

Fix patch:
diff --git a/README.md b/README.md
index 2bfc5664..09a534bb 100644
--- a/README.md
+++ b/README.md
@@ -200,6 +200,7 @@ You can find more projects and ecosystem tools in the [awesome-pest](https://git
 * [caith](https://github.com/Geobert/caith) (a dice roller crate)
 * [Melody](https://github.com/yoav-lavi/melody)
 * [json5-nodes](https://github.com/jlyonsmith/json5-nodes)
+* [prisma](https://github.com/prisma/prisma)
 
 ## Minimum Supported Rust Version (MSRV)
 
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
index b287fc8b..a8f1bee5 100644
--- a/debugger/Cargo.toml
+++ b/debugger/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_meta = { path = "../meta", version = "2.7.0" }
-pest_vm = { path = "../vm", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_meta = { path = "../meta", version = "2.7.1" }
+pest_vm = { path = "../vm", version = "2.7.1" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"
diff --git a/derive/Cargo.toml b/derive/Cargo.toml
index 4748cf8d..ee1b1d46 100644
--- a/derive/Cargo.toml
+++ b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.7.0", default-features = false }
-pest_generator = { path = "../generator", version = "2.7.0", default-features = false }
+pest = { path = "../pest", version = "2.7.1", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.1", default-features = false }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
index 8d11fad7..f4143042 100644
--- a/generator/Cargo.toml
+++ b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -20,8 +20,8 @@ not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
 grammar-extras = ["pest_meta/grammar-extras"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0", default-features = false }
-pest_meta = { path = "../meta", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.1" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
diff --git a/generator/src/generator.rs b/generator/src/generator.rs
index e36e9eb8..e41f9663 100644
--- a/generator/src/generator.rs
+++ b/generator/src/generator.rs
@@ -496,6 +496,26 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
                 })
             }
         }
+        #[cfg(feature = "grammar-extras")]
+        OptimizedExpr::RepOnce(expr) => {
+            let expr = generate_expr(*expr);
+
+            quote! {
+                state.sequence(|state| {
+                    #expr.and_then(|state| {
+                        state.repeat(|state| {
+                            state.sequence(|state| {
+                                super::hidden::skip(
+                                    state
+                                ).and_then(|state| {
+                                    #expr
+                                })
+                            })
+                        })
+                    })
+                })
+            }
+        }
         OptimizedExpr::Skip(strings) => {
             quote! {
                 let strings = [#(#strings),*];
@@ -635,6 +655,22 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
                 })
             }
         }
+        #[cfg(feature = "grammar-extras")]
+        OptimizedExpr::RepOnce(expr) => {
+            let expr = generate_expr_atomic(*expr);
+
+            quote! {
+                state.sequence(|state| {
+                    #expr.and_then(|state| {
+                        state.repeat(|state| {
+                            state.sequence(|state| {
+                                #expr
+                            })
+                        })
+                    })
+                })
+            }
+        }
         OptimizedExpr::Skip(strings) => {
             quote! {
                 let strings = [#(#strings),*];
diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
index f384220e..3b68b8af 100644
--- a/grammars/Cargo.toml
+++ b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_derive = { path = "../derive", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_derive = { path = "../derive", version = "2.7.1" }
 
 [dev-dependencies]
 criterion = "0.5"
diff --git a/meta/Cargo.toml b/meta/Cargo.toml
index 89b6f51a..2c7b1247 100644
--- a/meta/Cargo.toml
+++ b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
 once_cell = "1.8.0"
 
 [build-dependencies]
diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs
index b69163ed..65ddb8e9 100644
--- a/meta/src/optimizer/mod.rs
+++ b/meta/src/optimizer/mod.rs
@@ -70,11 +70,13 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
             Expr::Push(expr) => OptimizedExpr::Push(Box::new(to_optimized(*expr))),
             #[cfg(feature = "grammar-extras")]
             Expr::NodeTag(expr, tag) => OptimizedExpr::NodeTag(Box::new(to_optimized(*expr)), tag),
-            Expr::RepOnce(_)
-            | Expr::RepExact(..)
-            | Expr::RepMin(..)
-            | Expr::RepMax(..)
-            | Expr::RepMinMax(..) => unreachable!("No valid transformation to OptimizedRule"),
+            #[cfg(feature = "grammar-extras")]
+            Expr::RepOnce(expr) => OptimizedExpr::RepOnce(Box::new(to_optimized(*expr))),
+            #[cfg(not(feature = "grammar-extras"))]
+            Expr::RepOnce(_) => unreachable!("No valid transformation to OptimizedRule"),
+            Expr::RepExact(..) | Expr::RepMin(..) | Expr::RepMax(..) | Expr::RepMinMax(..) => {
+                unreachable!("No valid transformation to OptimizedRule")
+            }
         }
     }
 
@@ -135,6 +137,9 @@ pub enum OptimizedExpr {
     Opt(Box<OptimizedExpr>),
     /// Matches an expression zero or more times, e.g. `e*`
     Rep(Box<OptimizedExpr>),
+    /// Matches an expression one or more times, e.g. `e+`
+    #[cfg(feature = "grammar-extras")]
+    RepOnce(Box<OptimizedExpr>),
     /// Continues to match expressions until one of the strings in the `Vec` is found
     Skip(Vec<String>),
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs
index e3c360d9..3c97b1ff 100644
--- a/meta/src/optimizer/unroller.rs
+++ b/meta/src/optimizer/unroller.rs
@@ -15,6 +15,7 @@ pub fn unroll(rule: Rule) -> Rule {
         name,
         ty,
         expr: expr.map_bottom_up(|expr| match expr {
+            #[cfg(not(feature = "grammar-extras"))]
             Expr::RepOnce(expr) => Expr::Seq(expr.clone(), Box::new(Expr::Rep(expr))),
             Expr::RepExact(expr, num) => (1..num + 1)
                 .map(|_| *expr.clone())
diff --git a/pest/Cargo.toml b/pest/Cargo.toml
index f6afa2ac..cf3cd6a6 100644
--- a/pest/Cargo.toml
+++ b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
index 7a10d498..ffe6ea65 100644
--- a/vm/Cargo.toml
+++ b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_meta = { path = "../meta", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_meta = { path = "../meta", version = "2.7.1" }
 
 [features]
 grammar-extras = ["pest_meta/grammar-extras"]
\ No newline at end of file
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
index 3755a725..dddf07a2 100644
--- a/vm/src/lib.rs
+++ b/vm/src/lib.rs
@@ -224,6 +224,17 @@ impl Vm {
                     })
                 })
             }),
+            #[cfg(feature = "grammar-extras")]
+            OptimizedExpr::RepOnce(ref expr) => state.sequence(|state| {
+                self.parse_expr(expr, state).and_then(|state| {
+                    state.repeat(|state| {
+                        state.sequence(|state| {
+                            self.skip(state)
+                                .and_then(|state| self.parse_expr(expr, state))
+                        })
+                    })
+                })
+            }),
             OptimizedExpr::Push(ref expr) => state.stack_push(|state| self.parse_expr(expr, state)),
             OptimizedExpr::Skip(ref strings) => state.skip_until(
                 &strings


Test patch:
diff --git a/derive/tests/oneormore.pest b/derive/tests/oneormore.pest
new file mode 100644
index 00000000..4655101f
--- /dev/null
+++ b/derive/tests/oneormore.pest
@@ -0,0 +1,5 @@
+WHITESPACE = _{ " " | "\r" | "\n" | "\t" }
+
+
+identifier = { (ASCII_ALPHA | "_")+ }
+assign = { identifier ~ "<-" ~ identifier }
diff --git a/derive/tests/oneormore.rs b/derive/tests/oneormore.rs
new file mode 100644
index 00000000..6bdb3c59
--- /dev/null
+++ b/derive/tests/oneormore.rs
@@ -0,0 +1,29 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec::Vec};
+
+#[cfg(feature = "grammar-extras")]
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/oneormore.pest"]
+pub struct OneOrMoreParser;
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+pub fn test_one_or_more() {
+    let result = OneOrMoreParser::parse(Rule::assign, "k <- b\n");
+    assert!(result.is_ok());
+    let mut pairs = result.unwrap();
+    let pair = pairs.next().unwrap();
+    assert_eq!(pair.as_rule(), Rule::assign);
+    let mut inner = pair.into_inner();
+    let lhs = inner.next().unwrap();
+    assert_eq!(lhs.as_rule(), Rule::identifier);
+    assert_eq!(lhs.as_str(), "k");
+    let rhs = inner.next().unwrap();
+    assert_eq!(rhs.as_rule(), Rule::identifier);
+    assert_eq!(rhs.as_str(), "b");
+}