Repository: kevinmehall/rust-peg (language: rust)

Issue:
Generic input lifetime for grammar
Hi, I have a type `Tokens<'code>` in my code that wants to use `rust-peg`'s `parser!` proc-macro, however I can't figure out how to do something like the following:


```rust
peg::parser! {
    grammar feeny<'a>() for Tokens<'a> {
        // snip
    }
}
```


Is there a way to do this?


Discussion / hints:
Not currently. There is support for type and lifetime parameters on rules, but not on the grammar as a whole.

Implementing it would take adding `rust_ty_params()` to the `peg_grammar` rule in [the meta-grammar](https://github.com/kevinmehall/rust-peg/blob/master/peg-macros/grammar.rustpeg), and then threading that through to the right places in the output.
Is this particularly difficult? I'd be willing to do it with a small bit of mentoring

Fix patch:
diff --git a/peg-macros/ast.rs b/peg-macros/ast.rs
index 9db1fc8..396861a 100644
--- a/peg-macros/ast.rs
+++ b/peg-macros/ast.rs
@@ -5,6 +5,7 @@ pub struct Grammar {
     pub doc: Option<TokenStream>,
     pub visibility: Option<TokenStream>,
     pub name: Ident,
+    pub ty_params: Option<Vec<TokenStream>>,
     pub args: Vec<(Ident, TokenStream)>,
     pub items: Vec<Item>,
     pub input_type: TokenStream,
diff --git a/peg-macros/grammar.rs b/peg-macros/grammar.rs
index 8a0e3bd..d7b9db1 100644
--- a/peg-macros/grammar.rs
+++ b/peg-macros/grammar.rs
@@ -84,33 +84,31 @@ pub mod peg {
                                         __parse_IDENT(__input, __state, __err_state, __pos);
                                     match __seq_res {
                                         ::peg::RuleResult::Matched(__pos, name) => {
-                                            let __seq_res = __parse_grammar_args(
+                                            let __seq_res = match __parse_rust_ty_params(
                                                 __input,
                                                 __state,
                                                 __err_state,
                                                 __pos,
-                                            );
+                                            ) {
+                                                ::peg::RuleResult::Matched(__newpos, __value) => {
+                                                    ::peg::RuleResult::Matched(
+                                                        __newpos,
+                                                        Some(__value),
+                                                    )
+                                                }
+                                                ::peg::RuleResult::Failed => {
+                                                    ::peg::RuleResult::Matched(__pos, None)
+                                                }
+                                            };
                                             match __seq_res {
-                                                ::peg::RuleResult::Matched(__pos, args) => {
-                                                    match ::peg::ParseLiteral::parse_string_literal(
-                                                        __input, __pos, "for",
-                                                    ) {
-                                                        ::peg::RuleResult::Matched(
-                                                            __pos,
-                                                            __val,
-                                                        ) => {
-                                                            let __seq_res = {
-                                                                let str_start = __pos;
-                                                                match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
-                                                            };
-                                                            match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
-                                                        }
-                                                        ::peg::RuleResult::Failed => {
-                                                            __err_state
-                                                                .mark_failure(__pos, "\"for\"");
-                                                            ::peg::RuleResult::Failed
-                                                        }
-                                                    }
+                                                ::peg::RuleResult::Matched(__pos, ty_params) => {
+                                                    let __seq_res = __parse_grammar_args(
+                                                        __input,
+                                                        __state,
+                                                        __err_state,
+                                                        __pos,
+                                                    );
+                                                    match __seq_res { :: peg :: RuleResult :: Matched (__pos , args) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "for") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let str_start = __pos ; match match __parse_rust_type (__input , __state , __err_state , __pos) { :: peg :: RuleResult :: Matched (pos , _) => :: peg :: RuleResult :: Matched (pos , ()) , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } { :: peg :: RuleResult :: Matched (__newpos , _) => { :: peg :: RuleResult :: Matched (__newpos , :: peg :: ParseSlice :: parse_slice (__input , str_start , __newpos)) } , :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , input_type) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "{") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = { let mut __repeat_pos = __pos ; let mut __repeat_value = vec ! () ; loop { let __pos = __repeat_pos ; let __step_res = __parse_item (__input , __state , __err_state , __pos) ; match __step_res { :: peg :: RuleResult :: Matched (__newpos , __value) => { __repeat_pos = __newpos ; __repeat_value . push (__value) ; } , :: peg :: RuleResult :: Failed => { break ; } } } :: peg :: RuleResult :: Matched (__repeat_pos , __repeat_value) } ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , items) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "}") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { Grammar { doc , visibility , name , ty_params , args , input_type , items } }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"}\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"{\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"for\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
                                                 }
                                                 ::peg::RuleResult::Failed => {
                                                     ::peg::RuleResult::Failed
diff --git a/peg-macros/grammar.rustpeg b/peg-macros/grammar.rustpeg
index 67c0667..f684bd3 100644
--- a/peg-macros/grammar.rustpeg
+++ b/peg-macros/grammar.rustpeg
@@ -6,8 +6,8 @@ use crate::tokens::FlatTokenStream;
 use proc_macro2::{ TokenStream, Ident, Group, Literal, Delimiter, Span };
 
 pub rule peg_grammar() -> Grammar
-    = doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
-        { Grammar { doc, visibility, name, args, input_type, items } }
+    = doc:rust_doc_comment() visibility:rust_visibility() "grammar" name:IDENT() ty_params:rust_ty_params()? args:grammar_args() "for" input_type:$(rust_type()) "{" items:item()* "}"
+        { Grammar { doc, visibility, name, ty_params, args, input_type, items } }
 
 rule grammar_args() -> Vec<(Ident, TokenStream)>
     = "(" args:((i:IDENT() ":" t:$(rust_type()) { (i, t) })**",") ","? ")" { args }
diff --git a/peg-macros/translate.rs b/peg-macros/translate.rs
index 8bfe5b7..6476671 100644
--- a/peg-macros/translate.rs
+++ b/peg-macros/translate.rs
@@ -49,6 +49,7 @@ fn extra_args_call(grammar: &Grammar) -> TokenStream {
 struct Context<'a> {
     rules: &'a HashMap<String, &'a Rule>,
     rules_from_args: HashSet<String>,
+    extra_ty_params: &'a Option<Vec<TokenStream>>,
     extra_args_call: TokenStream,
     extra_args_def: TokenStream,
 }
@@ -62,6 +63,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
     let context = &Context {
         rules: &analysis.rules,
         rules_from_args: HashSet::new(),
+        extra_ty_params: &grammar.ty_params,
         extra_args_call: extra_args_call(grammar),
         extra_args_def: extra_args_def(grammar),
     };
@@ -107,6 +109,7 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
 
     let doc = &grammar.doc;
     let input_type = &grammar.input_type;
+    let input_ty_params = ty_params_slice(&grammar.ty_params);
     let visibility = &grammar.visibility;
 
     let mut errors = Vec::new();
@@ -124,8 +127,8 @@ pub(crate) fn compile_grammar(grammar: &Grammar) -> TokenStream {
         #visibility mod #name {
             #[allow(unused_imports)]
             use super::*;
-            type Input = #input_type;
-            type PositionRepr = <Input as ::peg::Parse>::PositionRepr;
+            type Input<#(#input_ty_params),*> = #input_type;
+            type PositionRepr<#(#input_ty_params),*> = <Input<#(#input_ty_params),*> as ::peg::Parse>::PositionRepr;
 
             #(#errors)*
             #(#items)*
@@ -165,14 +168,19 @@ fn make_parse_state(grammar: &Grammar) -> TokenStream {
     }
 }
 
-fn rule_params_list(rule: &Rule) -> Vec<TokenStream> {
+fn ty_params_slice(ty_params: &Option<Vec<TokenStream>>) -> &[TokenStream] {
+    ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[])
+}
+
+fn rule_params_list(context: &Context, rule: &Rule) -> Vec<TokenStream> {
+    let extra_ty_params = ty_params_slice(context.extra_ty_params);
     let span = rule.span.resolved_at(Span::mixed_site());
     rule.params.iter().map(|param| {
         let name = &param.name;
         match &param.ty {
             RuleParamTy::Rust(ty) => quote_spanned!{ span => #name: #ty },
             RuleParamTy::Rule(ty) => quote_spanned!{ span =>
-                #name: impl Fn(&'input Input, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
+                #name: impl Fn(&'input Input<#(#extra_ty_params),*>, &mut ParseState<'input>, &mut ::peg::error::ErrorState, usize) -> ::peg::RuleResult<#ty>
             },
         }
     }).collect()
@@ -184,7 +192,8 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
     let name = format_ident!("__parse_{}", rule.name, span=span);
     let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
     let result_used = rule.ret_type.is_some();
-    let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
+    let ty_params = ty_params_slice(&rule.ty_params);
+    let extra_ty_params = ty_params_slice(context.extra_ty_params);
 
     let mut context = context.clone();
     context
@@ -217,7 +226,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
 
     let extra_args_def = &context.extra_args_def;
 
-    let rule_params = rule_params_list(rule);
+    let rule_params = rule_params_list(&context, rule);
 
     if rule.cached {
         let cache_field = format_ident!("{}_cache", rule.name);
@@ -236,7 +245,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
         };
 
         quote_spanned! { span =>
-            fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
+            fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
                 #![allow(non_snake_case, unused)]
                 if let Some(entry) = __state.#cache_field.get(&__pos) {
                     #cache_trace
@@ -249,7 +258,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream {
         }
     } else {
         quote_spanned! { span =>
-            fn #name<'input #(, #ty_params)*>(__input: &'input Input, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
+            fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*>, __state: &mut ParseState<'input>, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> {
                 #![allow(non_snake_case, unused)]
                 #wrapped_body
             }
@@ -264,8 +273,9 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
     let ret_ty = rule.ret_type.clone().unwrap_or_else(|| quote!(()));
     let visibility = &rule.visibility;
     let parse_fn = format_ident!("__parse_{}", rule.name.to_string(), span = name.span());
-    let ty_params = rule.ty_params.as_ref().map(|x| &x[..]).unwrap_or(&[]);
-    let rule_params = rule_params_list(rule);
+    let ty_params = ty_params_slice(&rule.ty_params);
+    let extra_ty_params = ty_params_slice(context.extra_ty_params);
+    let rule_params = rule_params_list(context, rule);
     let rule_params_call: Vec<TokenStream> = rule
         .params
         .iter()
@@ -280,7 +290,7 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream {
 
     quote_spanned! { span =>
         #doc
-        #visibility fn #name<'input #(, #ty_params)*>(__input: &'input Input #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr>> {
+        #visibility fn #name<'input #(, #extra_ty_params)* #(, #ty_params)*>(__input: &'input Input<#(#extra_ty_params),*> #extra_args_def #(, #rule_params)*) -> ::std::result::Result<#ret_ty, ::peg::error::ParseError<PositionRepr<#(#extra_ty_params),*>>> {
             #![allow(non_snake_case, unused)]
 
             let mut __err_state = ::peg::error::ErrorState::new(::peg::Parse::start(__input));


Test patch:
diff --git a/tests/run-pass/lifetimes.rs b/tests/run-pass/lifetimes.rs
new file mode 100644
index 0000000..c6d2fec
--- /dev/null
+++ b/tests/run-pass/lifetimes.rs
@@ -0,0 +1,24 @@
+#[derive(Clone)]
+pub struct Token<'text>(&'text str);
+
+peg::parser!{
+    grammar tokenparser<'t>() for [Token<'t>] {
+        pub rule program() -> Vec<&'t str> = list()
+
+        // add this indirection to ensure that rule args work with a global lifetime
+        rule commasep<T>(x: rule<T>) -> Vec<T> = v:(x() ** [Token(",")]) [Token(",")]? { v }
+
+        rule list() -> Vec<&'t str> = [Token("(")] l:commasep(<string()>) [Token(")")] { l }
+        rule string() -> &'t str = [Token(inner)] { inner }
+    }
+}
+
+fn main() {
+    let input = "(one,two)";
+    assert_eq!(
+        tokenparser::program(
+            &[Token(&input[0..1]), Token(&input[1..4]), Token(&input[4..5]), Token(&input[5..8]), Token(&input[8..9])],
+        ),
+        Ok(vec!["one", "two"])
+    );
+}