Repository: pest-parser/pest
Pull request #871 (2023-06-20 11:16:42)

feature-guard the new grammar features with "grammar-extras"
this is to address potential semver issues due to Cargo dependency intermixing, see: https://github.com/pest-parser/pest/issues/849

So, for existing users who depend on 2.X in the dependency tree, 2.7.0 pest-meta should work with 2.5.6 and below pest-generator etc. People who need the new grammar features can do so by specifying the `grammar-extras` feature in Cargo.toml:
```
...
pest_derive = {version = "2.7", features = ["grammar-extras"]}
```


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ed020e6..d239dbb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
           kind: check
           toolchain: 1.64.0
       - name: cargo check
-        run: cargo check --all --features pretty-print,const_prec_climber,memchr --all-targets
+        run: cargo check --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets
 
   testing:
     name: Unit, Style, and Lint Testing
@@ -44,9 +44,9 @@ jobs:
       - name: cargo fmt
         run: cargo fmt --all -- --check
       - name: cargo clippy
-        run: cargo clippy --all --features pretty-print,const_prec_climber,memchr --all-targets -- -Dwarnings
+        run: cargo clippy --all --features pretty-print,const_prec_climber,memchr,grammar-extras --all-targets -- -Dwarnings
       - name: cargo test
-        run: cargo test --all --features pretty-print,const_prec_climber,memchr --release
+        run: cargo test --all --features pretty-print,const_prec_climber,memchr,grammar-extras --release
       - name: cargo test (ignored)
         run: cargo test -p pest_grammars --lib --verbose --release -- --ignored tests::toml_handles_deep_nesting_unstable
 
@@ -64,7 +64,7 @@ jobs:
           kind: check
           toolchain: 1.64.0
       - name: cargo doc
-        run: cargo doc --all --features pretty-print,const_prec_climber,memchr
+        run: cargo doc --all --features pretty-print,const_prec_climber,memchr,grammar-extras
 
   dependency:
     name: Minimal Versions Testing
@@ -101,7 +101,7 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage
-        run: cargo llvm-cov --features pretty-print,const_prec_climber,memchr --workspace --lcov --output-path lcov.info
+        run: cargo llvm-cov --features pretty-print,const_prec_climber,memchr,grammar-extras --workspace --lcov --output-path lcov.info
       - name: Upload Results to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -140,7 +140,7 @@ jobs:
         with:
           kind: check
           components: rust-src
-          toolchain: nightly-2022-11-20 # upgrade this regularly
+          toolchain: nightly-2023-03-20 # upgrade this regularly
       - name: check no_std compatibility
         run: cd pest && cargo build -j1 -Z build-std=core,alloc --no-default-features --target x86_64-unknown-linux-gnu
 
@@ -156,7 +156,7 @@ jobs:
         id: setup
         with:
           kind: check
-          toolchain: nightly-2022-11-20
+          toolchain: nightly-2023-03-20
           tools: cargo-semver-checks
       - name: check semver compatibility
         shell: bash
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
index c0b3c6a..b287fc8 100644
--- a/debugger/Cargo.toml
+++ b/debugger/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
-pest_meta = { path = "../meta", version = "2.6.1" }
-pest_vm = { path = "../vm", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
+pest_meta = { path = "../meta", version = "2.7.0" }
+pest_vm = { path = "../vm", version = "2.7.0" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"
diff --git a/derive/Cargo.toml b/derive/Cargo.toml
index 26fbc5c..4748cf8 100644
--- a/derive/Cargo.toml
+++ b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -21,8 +21,9 @@ proc-macro = true
 default = ["std"]
 std = ["pest/std", "pest_generator/std"]
 not-bootstrap-in-src = ["pest_generator/not-bootstrap-in-src"]
+grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.6.1", default-features = false }
-pest_generator = { path = "../generator", version = "2.6.1", default-features = false }
+pest = { path = "../pest", version = "2.7.0", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.0", default-features = false }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
index f9f36de..8d11fad 100644
--- a/generator/Cargo.toml
+++ b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -17,10 +17,11 @@ rust-version = "1.60"
 default = ["std"]
 std = ["pest/std"]
 not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
+grammar-extras = ["pest_meta/grammar-extras"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1", default-features = false }
-pest_meta = { path = "../meta", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.0" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
diff --git a/generator/src/generator.rs b/generator/src/generator.rs
index 9970ca8..e36e9eb 100644
--- a/generator/src/generator.rs
+++ b/generator/src/generator.rs
@@ -517,6 +517,7 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        #[cfg(feature = "grammar-extras")]
         OptimizedExpr::NodeTag(expr, tag) => {
             let expr = generate_expr(*expr);
             quote! {
@@ -655,6 +656,7 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
                 state.restore_on_err(|state| #expr)
             }
         }
+        #[cfg(feature = "grammar-extras")]
         OptimizedExpr::NodeTag(expr, tag) => {
             let expr = generate_expr_atomic(*expr);
             quote! {
diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
index a19243b..f384220 100644
--- a/grammars/Cargo.toml
+++ b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
-pest_derive = { path = "../derive", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
+pest_derive = { path = "../derive", version = "2.7.0" }
 
 [dev-dependencies]
 criterion = "0.5"
diff --git a/meta/Cargo.toml b/meta/Cargo.toml
index f5a3be9..89b6f51 100644
--- a/meta/Cargo.toml
+++ b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
 once_cell = "1.8.0"
 
 [build-dependencies]
@@ -25,4 +25,5 @@ cargo = { version = "0.70", optional = true }
 
 [features]
 default = []
-not-bootstrap-in-src = ["dep:cargo"]
\ No newline at end of file
+not-bootstrap-in-src = ["dep:cargo"]
+grammar-extras = []
\ No newline at end of file
diff --git a/meta/src/ast.rs b/meta/src/ast.rs
index 3a6a5b7..d731683 100644
--- a/meta/src/ast.rs
+++ b/meta/src/ast.rs
@@ -95,6 +95,7 @@ pub enum Expr {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<Expr>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<Expr>, String),
 }
 
@@ -166,6 +167,7 @@ impl Expr {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::Push(mapped)
                 }
+                #[cfg(feature = "grammar-extras")]
                 Expr::NodeTag(expr, tag) => {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::NodeTag(mapped, tag)
@@ -237,6 +239,7 @@ impl Expr {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::Push(mapped)
                 }
+                #[cfg(feature = "grammar-extras")]
                 Expr::NodeTag(expr, tag) => {
                     let mapped = Box::new(map_internal(*expr, f));
                     Expr::NodeTag(mapped, tag)
@@ -290,8 +293,11 @@ impl ExprTopDownIterator {
             | Expr::RepMax(expr, _)
             | Expr::RepMinMax(expr, ..)
             | Expr::Opt(expr)
-            | Expr::Push(expr)
-            | Expr::NodeTag(expr, _) => {
+            | Expr::Push(expr) => {
+                self.next = Some(*expr);
+            }
+            #[cfg(feature = "grammar-extras")]
+            Expr::NodeTag(expr, _) => {
                 self.next = Some(*expr);
             }
             _ => {
diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs
index 747611d..b69163e 100644
--- a/meta/src/optimizer/mod.rs
+++ b/meta/src/optimizer/mod.rs
@@ -68,6 +68,7 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
             Expr::Rep(expr) => OptimizedExpr::Rep(Box::new(to_optimized(*expr))),
             Expr::Skip(strings) => OptimizedExpr::Skip(strings),
             Expr::Push(expr) => OptimizedExpr::Push(Box::new(to_optimized(*expr))),
+            #[cfg(feature = "grammar-extras")]
             Expr::NodeTag(expr, tag) => OptimizedExpr::NodeTag(Box::new(to_optimized(*expr)), tag),
             Expr::RepOnce(_)
             | Expr::RepExact(..)
@@ -139,6 +140,7 @@ pub enum OptimizedExpr {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<OptimizedExpr>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<OptimizedExpr>, String),
     /// Restores an expression's checkpoint
     RestoreOnErr(Box<OptimizedExpr>),
diff --git a/meta/src/parser.rs b/meta/src/parser.rs
index ee39de6..a4e8d1b 100644
--- a/meta/src/parser.rs
+++ b/meta/src/parser.rs
@@ -20,7 +20,6 @@ use pest::{Parser, Position, Span};
 use crate::ast::{Expr, Rule as AstRule, RuleType};
 use crate::validator;
 
-/// TODO: fix the generator to at least add explicit lifetimes
 #[allow(missing_docs, unused_qualifications)]
 mod grammar {
     #[cfg(not(feature = "not-bootstrap-in-src"))]
@@ -163,6 +162,7 @@ pub enum ParserExpr<'i> {
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`
     Push(Box<ParserNode<'i>>),
     /// Matches an expression and assigns a label to it, e.g. #label = exp
+    #[cfg(feature = "grammar-extras")]
     NodeTag(Box<ParserNode<'i>>, String),
 }
 
@@ -199,6 +199,7 @@ fn convert_node(node: ParserNode<'_>) -> Expr {
             Expr::RepMinMax(Box::new(convert_node(*node)), min, max)
         }
         ParserExpr::Push(node) => Expr::Push(Box::new(convert_node(*node))),
+        #[cfg(feature = "grammar-extras")]
         ParserExpr::NodeTag(node, tag) => Expr::NodeTag(Box::new(convert_node(*node)), tag),
     }
 }
@@ -337,7 +338,10 @@ fn consume_expr<'i>(
         mut pairs: Peekable<Pairs<'i, Rule>>,
         pratt: &PrattParser<Rule>,
     ) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
+        #[cfg(feature = "grammar-extras")]
         let (pair, tag_start) = get_node_tag(&mut pairs);
+        #[cfg(not(feature = "grammar-extras"))]
+        let (pair, _tag_start) = get_node_tag(&mut pairs);
 
         let node = match pair.as_rule() {
             Rule::opening_paren => {
@@ -632,6 +636,7 @@ fn consume_expr<'i>(
                 )?
             }
         };
+        #[cfg(feature = "grammar-extras")]
         if let Some((tag, start)) = tag_start {
             let span = start.span(&node.span.end_pos());
             Ok(ParserNode {
@@ -641,6 +646,8 @@ fn consume_expr<'i>(
         } else {
             Ok(node)
         }
+        #[cfg(not(feature = "grammar-extras"))]
+        Ok(node)
     }
 
     let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), pratt);
diff --git a/meta/src/validator.rs b/meta/src/validator.rs
index a365530..16d3879 100644
--- a/meta/src/validator.rs
+++ b/meta/src/validator.rs
@@ -326,9 +326,9 @@ fn is_non_progressing<'i>(
             min == 0 || is_non_progressing(&inner.expr, rules, trace)
         }
         ParserExpr::Push(ref inner) => is_non_progressing(&inner.expr, rules, trace),
-        ParserExpr::RepOnce(ref inner) | ParserExpr::NodeTag(ref inner, _) => {
-            is_non_progressing(&inner.expr, rules, trace)
-        }
+        ParserExpr::RepOnce(ref inner) => is_non_progressing(&inner.expr, rules, trace),
+        #[cfg(feature = "grammar-extras")]
+        ParserExpr::NodeTag(ref inner, _) => is_non_progressing(&inner.expr, rules, trace),
     }
 }
 
@@ -417,9 +417,11 @@ fn is_non_failing<'i>(
         //     @{EOI ~ ANY | ANY ~ SOI | &("A") ~ &("B") | 'z'..'a'}
         ParserExpr::NegPred(_) => false,
         ParserExpr::RepOnce(ref inner) => is_non_failing(&inner.expr, rules, trace),
-        ParserExpr::Push(ref inner)
-        | ParserExpr::NodeTag(ref inner, _)
-        | ParserExpr::PosPred(ref inner) => is_non_failing(&inner.expr, rules, trace),
+        ParserExpr::Push(ref inner) | ParserExpr::PosPred(ref inner) => {
+            is_non_failing(&inner.expr, rules, trace)
+        }
+        #[cfg(feature = "grammar-extras")]
+        ParserExpr::NodeTag(ref inner, _) => is_non_failing(&inner.expr, rules, trace),
     }
 }
 
@@ -933,19 +935,22 @@ mod tests {
             &HashMap::new(),
             &mut Vec::new()
         ));
-        let progressing = ParserExpr::NodeTag(progressing_node.clone(), "TAG".into());
-        let non_progressing = ParserExpr::NodeTag(non_progressing_node.clone(), "TAG".into());
+        #[cfg(feature = "grammar-extras")]
+        {
+            let progressing = ParserExpr::NodeTag(progressing_node.clone(), "TAG".into());
+            let non_progressing = ParserExpr::NodeTag(non_progressing_node.clone(), "TAG".into());
 
-        assert!(!is_non_progressing(
-            &progressing,
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
-        assert!(is_non_progressing(
-            &non_progressing,
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
+            assert!(!is_non_progressing(
+                &progressing,
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+            assert!(is_non_progressing(
+                &non_progressing,
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+        }
     }
 
     #[test]
@@ -1290,6 +1295,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature = "grammar-extras")]
     fn failing_non_zero_repetitions() {
         let failing = Box::new(ParserNode {
             expr: ParserExpr::NodeTag(
@@ -1540,16 +1546,19 @@ mod tests {
             &mut Vec::new()
         ));
 
-        assert!(!is_non_failing(
-            &ParserExpr::NodeTag(failing_node.clone(), "TAG".into()),
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
-        assert!(is_non_failing(
-            &ParserExpr::NodeTag(non_failing_node.clone(), "TAG".into()),
-            &HashMap::new(),
-            &mut Vec::new()
-        ));
+        #[cfg(feature = "grammar-extras")]
+        {
+            assert!(!is_non_failing(
+                &ParserExpr::NodeTag(failing_node.clone(), "TAG".into()),
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+            assert!(is_non_failing(
+                &ParserExpr::NodeTag(non_failing_node.clone(), "TAG".into()),
+                &HashMap::new(),
+                &mut Vec::new()
+            ));
+        }
 
         assert!(!is_non_failing(
             &ParserExpr::Push(failing_node.clone()),
diff --git a/pest/Cargo.toml b/pest/Cargo.toml
index 52e6406..f6afa2a 100644
--- a/pest/Cargo.toml
+++ b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
diff --git a/release.sh b/release.sh
index 5fedb10..73cc420 100755
--- a/release.sh
+++ b/release.sh
@@ -18,11 +18,11 @@ get_manifest_path() {
 
 publish() {
   echo "Publishing crate $1..."
-  if [ "${1}" == "pest" ]; then
+  if [ "${1}" = "pest" ] || [ "${1}" = "pest_grammars" ] || [ "${1}" = "pest_debugger" ]; then
     cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --all-features
   else
     # cannot publish with the `not-bootstrap-in-src` feature enabled
-    cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty
+    cargo publish --manifest-path "$(get_manifest_path "${1}")" --allow-dirty --features grammar-extras
   fi
   echo ""
 }
diff --git a/semvercheck.sh b/semvercheck.sh
index 8d48d83..df3748b 100755
--- a/semvercheck.sh
+++ b/semvercheck.sh
@@ -8,7 +8,7 @@ cargo run --package pest_bootstrap
 
 # current
 for crate in "pest_derive" "pest_generator" "pest_grammars" "pest_meta" "pest" "pest_vm" "pest_debugger"; do
-    cargo +nightly-2022-11-20 rustdoc -p $crate -- $RUSTDOC_LATE_FLAGS
+    cargo +nightly-2023-03-20 rustdoc -p $crate -- $RUSTDOC_LATE_FLAGS
     mv target/doc/$crate.json /tmp/current-$crate.json
 done
 
@@ -21,7 +21,7 @@ cargo clean
 cargo build --package pest_bootstrap
 cargo run --package pest_bootstrap
 for crate in "pest_derive" "pest_generator" "pest_grammars" "pest_meta" "pest" "pest_vm" "pest_debugger"; do
-    cargo +nightly-2022-11-20 rustdoc -p $crate -- $RUSTDOC_LATE_FLAGS
+    cargo +nightly-2023-03-20 rustdoc -p $crate -- $RUSTDOC_LATE_FLAGS
     mv target/doc/$crate.json /tmp/baseline-$crate.json
     echo "Checking $crate"
     cargo semver-checks check-release --current /tmp/current-$crate.json --baseline /tmp/baseline-$crate.json
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
index 6e926a7..7a10d49 100644
--- a/vm/Cargo.toml
+++ b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.6.1"
+version = "2.7.0"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,5 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.6.1" }
-pest_meta = { path = "../meta", version = "2.6.1" }
+pest = { path = "../pest", version = "2.7.0" }
+pest_meta = { path = "../meta", version = "2.7.0" }
+
+[features]
+grammar-extras = ["pest_meta/grammar-extras"]
\ No newline at end of file
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
index c46c0f7..3755a72 100644
--- a/vm/src/lib.rs
+++ b/vm/src/lib.rs
@@ -231,6 +231,7 @@ impl Vm {
                     .map(|state| state.as_str())
                     .collect::<Vec<&str>>(),
             ),
+            #[cfg(feature = "grammar-extras")]
             OptimizedExpr::NodeTag(ref expr, ref tag) => self
                 .parse_expr(expr, state)
                 .and_then(|state| state.tag_node(std::borrow::Cow::Owned(tag.clone()))),


Test patch:
diff --git a/derive/tests/implicit.rs b/derive/tests/implicit.rs
index 2d249ba..48c8802 100644
--- a/derive/tests/implicit.rs
+++ b/derive/tests/implicit.rs
@@ -9,6 +9,7 @@ extern crate alloc;
 extern crate pest;
 extern crate pest_derive;
 
+#[cfg(feature = "grammar-extras")]
 use pest::Parser;
 use pest_derive::Parser;
 
@@ -17,6 +18,7 @@ use pest_derive::Parser;
 struct TestImplicitParser;
 
 #[test]
+#[cfg(feature = "grammar-extras")]
 fn test_implicit_whitespace() {
     // this failed to parse due to a bug in the optimizer
     // see: https://github.com/pest-parser/pest/issues/762#issuecomment-1375374868