Repository: rcoh/angle-grinder
Pull request #154 (2021-09-02 16:05:20)

[lang] support regex option for parse operator
This change adds support for `parse regex`.  It also fixes handling of escape sequences in quoted strings since they weren't really working before.


Patch:
diff --git a/README.md b/README.md
index 6e63d94..c7a3ba2 100644
--- a/README.md
+++ b/README.md
@@ -193,6 +193,28 @@ expression will match _any_ whitespace character in the input text (eg. a litera
 ```
 ![parse.gif](/screen_shots/parse.gif)
 
+##### Parse Regex
+`parse regex "<regex-with-named-captures>" [from field] [nodrop]`: Match the
+input text against a regular expression and populate the record with the named
+captures.  Lines that don't match the pattern will be dropped unless `nodrop` is
+specified. By default, `parse` operates on the raw text of the message. With
+`from field_name`, parse will instead process input from a specific column.
+
+*Notes*:
+
+- Only named captures are supported.  If the regular expression includes any
+  unnamed captures, an error will be raised.
+- The [Rust regular expression syntax](https://docs.rs/regex/latest/regex/#syntax) is used.
+- Escape sequences do not require an extra backslash (i.e. `\w` works as-is).
+
+*Examples*:
+To parse the phrase "Hello, ...!" and capture the value of the "..." in the
+name field:
+
+```agrind
+* | parse regex "Hello, (?P<name>\w+)"
+```
+
 ##### Fields
 `fields [only|except|-|+] a, b`: Drop fields `a, b` or include only `a, b` depending on specified mode.
 
diff --git a/src/lang.rs b/src/lang.rs
index 27c177b..206b20f 100644
--- a/src/lang.rs
+++ b/src/lang.rs
@@ -4,12 +4,13 @@ use std::str;
 
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use nom::bytes::complete::escaped;
 use nom::combinator::not;
 use nom::multi::{fold_many0, fold_many1};
 use nom::sequence::{delimited, separated_pair};
 use nom::{
     branch::alt,
-    bytes::complete::{escaped, take_while, take_while1},
+    bytes::complete::{take, take_while, take_while1},
     character::complete::{anychar, digit1, multispace0, multispace1, none_of, satisfy},
     character::{is_alphabetic, is_alphanumeric},
     combinator::{eof, map, map_res, opt, peek, recognize},
@@ -17,7 +18,7 @@ use nom::{
     multi::{many0, many_till, separated_list0, separated_list1},
     number::complete::double,
     sequence::{pair, tuple},
-    IResult, Parser, Slice,
+    IResult, InputIter, Parser, Slice,
 };
 use nom_locate::position;
 use nom_locate::LocatedSpan;
@@ -364,9 +365,10 @@ where
 #[derive(Debug, PartialEq, Eq, Clone)]
 enum KeywordType {
     /// The keyword string should exactly match the input.
-    EXACT,
+    Exact,
     /// The keyword string can contain wildcards.
-    WILDCARD,
+    Wildcard,
+    Regex,
 }
 
 /// Represents a `keyword` search string.
@@ -376,12 +378,17 @@ pub struct Keyword(String, KeywordType);
 impl Keyword {
     /// Create a Keyword that will exactly match an input string.
     pub fn new_exact(str: String) -> Keyword {
-        Keyword(str, KeywordType::EXACT)
+        Keyword(str, KeywordType::Exact)
     }
 
     /// Create a Keyword that can contain wildcards
     pub fn new_wildcard(str: String) -> Keyword {
-        Keyword(str, KeywordType::WILDCARD)
+        Keyword(str, KeywordType::Wildcard)
+    }
+
+    /// Create a Keyword that can contain wildcards
+    pub fn new_regex(str: String) -> Keyword {
+        Keyword(str, KeywordType::Regex)
     }
 
     /// Test if this is an empty keyword string
@@ -391,10 +398,14 @@ impl Keyword {
 
     /// Convert this keyword to a `regex::Regex` object.
     pub fn to_regex(&self) -> regex::Regex {
+        if self.1 == KeywordType::Regex {
+            return regex::Regex::new(&self.0).unwrap();
+        }
+
         let mut regex_str = regex::escape(&self.0.replace("\\\"", "\"")).replace(" ", "\\s");
 
         regex_str.insert_str(0, "(?i)");
-        if self.1 == KeywordType::WILDCARD {
+        if self.1 == KeywordType::Wildcard {
             regex_str = regex_str.replace("\\*", "(.*?)");
             // If it ends with a star, we need to ensure we read until the end.
             if self.0.ends_with('*') {
@@ -1138,8 +1149,60 @@ fn req_ident(input: Span) -> IResult<Span, String> {
     .parse(input)
 }
 
+fn escaped_chars(input: Span) -> IResult<Span, Span> {
+    alt((
+        tag("\\"),
+        tag("r"),
+        tag("n"),
+        tag("t"),
+        tag("0"),
+        tag("'"),
+        tag("\""),
+        // Accept other characters and pass them through
+        // so the user doesn't have to do extra escaping
+        take(1usize),
+        // TODO handle unicode
+    ))
+    .parse(input)
+}
+
+fn escaped_str_transformer(input: Span) -> String {
+    let mut in_escape = false;
+
+    input
+        .fragment()
+        .iter_elements()
+        .fold(String::new(), |mut acc, ch| {
+            let was_in_escape = in_escape;
+            match ch {
+                '\\' if !in_escape => {
+                    in_escape = true;
+                }
+                '\\' if in_escape => acc.push('\\'),
+                't' if in_escape => acc.push('\t'),
+                'r' if in_escape => acc.push('\r'),
+                'n' if in_escape => acc.push('\n'),
+                '0' if in_escape => acc.push('\0'),
+                '\'' if in_escape => acc.push('\''),
+                '"' if in_escape => acc.push('"'),
+                unhandled_escape if in_escape => {
+                    // note: unknown escapes are passed through so that escape sequences in
+                    // regexes "just work"
+                    acc.push('\\');
+                    acc.push(unhandled_escape)
+                }
+                other => acc.push(other),
+            };
+            if was_in_escape {
+                in_escape = false;
+            }
+
+            acc
+        })
+}
+
 fn quoted_string(input: Span) -> IResult<Span, String> {
-    let sq_esc = escaped(none_of("\\\'"), '\\', tag("'"));
+    let sq_esc = escaped(none_of("\\\'"), '\\', escaped_chars);
     let sq_esc_or_empty = alt((sq_esc, tag("")));
     let single_quoted = expect_delimited(tag("'"), sq_esc_or_empty, tag("'"), |qc, r| {
         qc.report_error_for("unterminated single quoted string")
@@ -1148,7 +1211,7 @@ fn quoted_string(input: Span) -> IResult<Span, String> {
             .send_report()
     });
 
-    let dq_esc = escaped(none_of("\\\""), '\\', tag("\""));
+    let dq_esc = escaped(none_of("\\\""), '\\', escaped_chars);
     let dq_esc_or_empty = alt((dq_esc, tag("")));
     let double_quoted = expect_delimited(tag("\""), dq_esc_or_empty, tag("\""), |qc, r| {
         qc.report_error_for("unterminated double quoted string")
@@ -1158,7 +1221,7 @@ fn quoted_string(input: Span) -> IResult<Span, String> {
     });
 
     alt((single_quoted, double_quoted))
-        .map(|s: Span| s.to_string())
+        .map(escaped_str_transformer)
         .parse(input)
 }
 
@@ -1187,17 +1250,71 @@ fn var_list(input: Span) -> IResult<Span, Vec<String>> {
 fn parse(input: Span) -> IResult<Span, Positioned<InlineOperator>> {
     with_pos(
         tuple((
-            tag("parse").precedes(multispace1.precedes(req_quoted_string)),
+            tag("parse").precedes(multispace1),
+            opt(tag("regex").precedes(multispace1)),
+            with_pos(req_quoted_string),
             opt(multispace1.precedes(with_pos(pair(tag("from"), multispace1).precedes(expr)))),
-            tag("as").preceded_by(multispace1).precedes(var_list),
+            opt(with_pos(tag("as").preceded_by(multispace1).precedes(var_list))),
             opt(multispace1.precedes(with_pos(pair(tag("from"), multispace1).precedes(expr)))),
             opt(tag("nodrop").preceded_by(multispace1)).map(|nd| nd.is_some()),
         ))
-        .map(|(s, e1, fields, e2, no_drop)| InlineOperator::Parse {
-            pattern: Keyword::new_wildcard(s),
-            fields,
-            input_column: (e1, e2),
-            no_drop,
+        .map(|(_p, is_regex, s, from_col_before, user_fields_opt, from_col_after, no_drop)| {
+            let (pattern, fields) = if is_regex.is_some() {
+                let named_fields: Vec<String> = match regex::Regex::new(&s.value) {
+                    Err(regex_err) => {
+                        input
+                            .extra
+                            .report_error_for("invalid regular expression")
+                            .with_code_range(s.range, format!("{}", regex_err))
+                            .send_report();
+                        Vec::new()
+                    }
+                    Ok(re) => {
+                        let named_caps: Vec<String> = re
+                            .capture_names()
+                            .into_iter()
+                            .flatten()
+                            .map_into()
+                            .collect();
+                        let unnamed_capture_count = re.captures_len() - 1 - named_caps.len();
+
+                        if unnamed_capture_count > 0 {
+                            input
+                                .extra
+                                .report_error_for("regular expression for the parse operator cannot have unnamed captures")
+                                .with_code_range(s.range, format!(
+                                    "contains {} unnamed capture(s)", unnamed_capture_count))
+                                .with_resolution("name the capture groups by adding -- ?P<capname> -- right after the left parenthesis")
+                                .send_report();
+                            Vec::new()
+                        } else if let Some(user_fields) = user_fields_opt {
+                            input
+                                .extra
+                                .report_error_for("the parse regex operator does not support an 'as' clause")
+                                .with_code_range(user_fields.range, "remove this")
+                                .with_resolution("remove the 'as' clause, the named capture groups in the regular expression determine the output column")
+                                .send_report();
+                            Vec::new()
+                        } else {
+                            named_caps
+                        }
+                    }
+                };
+
+                (Keyword::new_regex(s.value), named_fields)
+            } else {
+                (
+                    Keyword::new_wildcard(s.value),
+                    user_fields_opt.map(|user_fields| user_fields.value).unwrap_or_default(),
+                )
+            };
+
+            InlineOperator::Parse {
+                pattern,
+                fields,
+                input_column: (from_col_before, from_col_after),
+                no_drop,
+            }
         }),
     )
     .terminated(expect_pipe(
@@ -1718,6 +1835,87 @@ mod tests {
 
     #[test]
     fn parse_parses() {
+        check_query(
+            r#"* | parse regex "(?P<abc>def)""#,
+            expect![[r#"
+                Query {
+                    search: And(
+                        [],
+                    ),
+                    operators: [
+                        Inline(
+                            Positioned {
+                                range: 4..30,
+                                value: Parse {
+                                    pattern: Keyword(
+                                        "(?P<abc>def)",
+                                        Regex,
+                                    ),
+                                    fields: [
+                                        "abc",
+                                    ],
+                                    input_column: (
+                                        None,
+                                        None,
+                                    ),
+                                    no_drop: false,
+                                },
+                            },
+                        ),
+                    ],
+                }
+            "#]],
+        );
+        check_query(
+            r#"* | parse regex "(?<abc>def""#,
+            expect![[r#"
+                Query {
+                    search: And(
+                        [],
+                    ),
+                    operators: [],
+                }
+                error: invalid regular expression
+                  |
+                1 | * | parse regex "(?<abc>def"
+                  |                 ^^^^^^^^^^^^ regex parse error:
+                    (?<abc>def
+                      ^
+                error: unrecognized flag
+                  |"#]],
+        );
+        check_query(
+            r#"* | parse regex "(?P<abc>def)" as v"#,
+            expect![[r#"
+                Query {
+                    search: And(
+                        [],
+                    ),
+                    operators: [],
+                }
+                error: the parse regex operator does not support an 'as' clause
+                  |
+                1 | * | parse regex "(?P<abc>def)" as v
+                  |                               ^^^^^ remove this
+                  |
+                  = help: remove the 'as' clause, the named capture groups in the regular expression determine the output column"#]],
+        );
+        check_query(
+            r#"* | parse regex "([0-9])""#,
+            expect![[r#"
+                Query {
+                    search: And(
+                        [],
+                    ),
+                    operators: [],
+                }
+                error: regular expression for the parse operator cannot have unnamed captures
+                  |
+                1 | * | parse regex "([0-9])"
+                  |                 ^^^^^^^^^ contains 1 unnamed capture(s)
+                  |
+                  = help: name the capture groups by adding -- ?P<capname> -- right after the left parenthesis"#]],
+        );
         check_query(
             r#"* | parse "[key=*]" as v"#,
             expect![[r#"
@@ -1732,7 +1930,7 @@ mod tests {
                                 value: Parse {
                                     pattern: Keyword(
                                         "[key=*]",
-                                        WILDCARD,
+                                        Wildcard,
                                     ),
                                     fields: [
                                         "v",
@@ -1763,7 +1961,7 @@ mod tests {
                                 value: Parse {
                                     pattern: Keyword(
                                         "[key=*]",
-                                        WILDCARD,
+                                        Wildcard,
                                     ),
                                     fields: [
                                         "v",
@@ -1783,34 +1981,34 @@ mod tests {
         check_query(
             r#"* | parse "[key=*][val=*]" as k,v nodrop"#,
             expect![[r#"
-            Query {
-                search: And(
-                    [],
-                ),
-                operators: [
-                    Inline(
-                        Positioned {
-                            range: 4..40,
-                            value: Parse {
-                                pattern: Keyword(
-                                    "[key=*][val=*]",
-                                    WILDCARD,
-                                ),
-                                fields: [
-                                    "k",
-                                    "v",
-                                ],
-                                input_column: (
-                                    None,
-                                    None,
-                                ),
-                                no_drop: true,
-                            },
-                        },
+                Query {
+                    search: And(
+                        [],
                     ),
-                ],
-            }
-        "#]],
+                    operators: [
+                        Inline(
+                            Positioned {
+                                range: 4..40,
+                                value: Parse {
+                                    pattern: Keyword(
+                                        "[key=*][val=*]",
+                                        Wildcard,
+                                    ),
+                                    fields: [
+                                        "k",
+                                        "v",
+                                    ],
+                                    input_column: (
+                                        None,
+                                        None,
+                                    ),
+                                    no_drop: true,
+                                },
+                            },
+                        ),
+                    ],
+                }
+            "#]],
         );
         check_query(
             r#"* | parse "[key=*]" from field as v"#,
@@ -1826,7 +2024,7 @@ mod tests {
                                 value: Parse {
                                     pattern: Keyword(
                                         "[key=*]",
-                                        WILDCARD,
+                                        Wildcard,
                                     ),
                                     fields: [
                                         "v",
@@ -1867,7 +2065,7 @@ mod tests {
                                 value: Parse {
                                     pattern: Keyword(
                                         "[key=*]",
-                                        WILDCARD,
+                                        Wildcard,
                                     ),
                                     fields: [
                                         "v",
@@ -1916,38 +2114,38 @@ mod tests {
         check_query(
             "abc",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Keyword(
+                Query {
+                    search: And(
+                        [
                             Keyword(
-                                "abc",
-                                WILDCARD,
+                                Keyword(
+                                    "abc",
+                                    Wildcard,
+                                ),
                             ),
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
         check_query(
             "one-two-three",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Keyword(
+                Query {
+                    search: And(
+                        [
                             Keyword(
-                                "one-two-three",
-                                WILDCARD,
+                                Keyword(
+                                    "one-two-three",
+                                    Wildcard,
+                                ),
                             ),
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
     }
 
@@ -2458,70 +2656,70 @@ mod tests {
         check_query(
             " abc def \"*ghi*\" ",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Keyword(
+                Query {
+                    search: And(
+                        [
                             Keyword(
-                                "abc",
-                                WILDCARD,
+                                Keyword(
+                                    "abc",
+                                    Wildcard,
+                                ),
                             ),
-                        ),
-                        Keyword(
                             Keyword(
-                                "def",
-                                WILDCARD,
+                                Keyword(
+                                    "def",
+                                    Wildcard,
+                                ),
                             ),
-                        ),
-                        Keyword(
                             Keyword(
-                                "*ghi*",
-                                EXACT,
+                                Keyword(
+                                    "*ghi*",
+                                    Exact,
+                                ),
                             ),
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
         check_query(
             "(abc AND def) OR xyz",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Or(
-                            [
-                                And(
-                                    [
-                                        Keyword(
+                Query {
+                    search: And(
+                        [
+                            Or(
+                                [
+                                    And(
+                                        [
                                             Keyword(
-                                                "abc",
-                                                WILDCARD,
+                                                Keyword(
+                                                    "abc",
+                                                    Wildcard,
+                                                ),
                                             ),
-                                        ),
-                                        Keyword(
                                             Keyword(
-                                                "def",
-                                                WILDCARD,
+                                                Keyword(
+                                                    "def",
+                                                    Wildcard,
+                                                ),
                                             ),
-                                        ),
-                                    ],
-                                ),
-                                Keyword(
+                                        ],
+                                    ),
                                     Keyword(
-                                        "xyz",
-                                        WILDCARD,
+                                        Keyword(
+                                            "xyz",
+                                            Wildcard,
+                                        ),
                                     ),
-                                ),
-                            ],
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                                ],
+                            ),
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
     }
 
@@ -2776,40 +2974,40 @@ mod tests {
         check_query(
             "abc OR (def OR xyz)",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Or(
-                            [
-                                Keyword(
+                Query {
+                    search: And(
+                        [
+                            Or(
+                                [
                                     Keyword(
-                                        "abc",
-                                        WILDCARD,
-                                    ),
-                                ),
-                                Or(
-                                    [
                                         Keyword(
+                                            "abc",
+                                            Wildcard,
+                                        ),
+                                    ),
+                                    Or(
+                                        [
                                             Keyword(
-                                                "def",
-                                                WILDCARD,
+                                                Keyword(
+                                                    "def",
+                                                    Wildcard,
+                                                ),
                                             ),
-                                        ),
-                                        Keyword(
                                             Keyword(
-                                                "xyz",
-                                                WILDCARD,
+                                                Keyword(
+                                                    "xyz",
+                                                    Wildcard,
+                                                ),
                                             ),
-                                        ),
-                                    ],
-                                ),
-                            ],
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                                        ],
+                                    ),
+                                ],
+                            ),
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
     }
 
@@ -2818,60 +3016,114 @@ mod tests {
         check_query(
             "NOT abc",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        Not(
-                            Keyword(
+                Query {
+                    search: And(
+                        [
+                            Not(
                                 Keyword(
-                                    "abc",
-                                    WILDCARD,
+                                    Keyword(
+                                        "abc",
+                                        Wildcard,
+                                    ),
                                 ),
                             ),
-                        ),
-                    ],
-                ),
-                operators: [],
-            }
-        "#]],
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
         );
         check_query(
             "(error OR warn) AND NOT hide",
             expect![[r#"
-            Query {
-                search: And(
-                    [
-                        And(
-                            [
-                                Or(
-                                    [
-                                        Keyword(
+                Query {
+                    search: And(
+                        [
+                            And(
+                                [
+                                    Or(
+                                        [
                                             Keyword(
-                                                "error",
-                                                WILDCARD,
+                                                Keyword(
+                                                    "error",
+                                                    Wildcard,
+                                                ),
                                             ),
-                                        ),
+                                            Keyword(
+                                                Keyword(
+                                                    "warn",
+                                                    Wildcard,
+                                                ),
+                                            ),
+                                        ],
+                                    ),
+                                    Not(
                                         Keyword(
                                             Keyword(
-                                                "warn",
-                                                WILDCARD,
+                                                "hide",
+                                                Wildcard,
                                             ),
                                         ),
-                                    ],
-                                ),
-                                Not(
-                                    Keyword(
-                                        Keyword(
-                                            "hide",
-                                            WILDCARD,
+                                    ),
+                                ],
+                            ),
+                        ],
+                    ),
+                    operators: [],
+                }
+            "#]],
+        );
+    }
+
+    #[test]
+    fn quoted_strings() {
+        check_query(
+            "* | 'abc\\tdef' as value",
+            expect![[r#"
+                Query {
+                    search: And(
+                        [],
+                    ),
+                    operators: [
+                        Inline(
+                            Positioned {
+                                range: 4..23,
+                                value: FieldExpression {
+                                    value: Value(
+                                        Str(
+                                            "abc\tdef",
                                         ),
                                     ),
-                                ),
-                            ],
+                                    name: "value",
+                                },
+                            },
                         ),
                     ],
+                }
+            "#]],
+        );
+        check_query(
+            "* | 'abc\\d+def' as v",
+            expect![[r#"
+            Query {
+                search: And(
+                    [],
                 ),
-                operators: [],
+                operators: [
+                    Inline(
+                        Positioned {
+                            range: 4..20,
+                            value: FieldExpression {
+                                value: Value(
+                                    Str(
+                                        "abc\\d+def",
+                                    ),
+                                ),
+                                name: "v",
+                            },
+                        },
+                    ),
+                ],
             }
         "#]],
         );


Test patch:
diff --git a/tests/structured_tests/parse_regex.toml b/tests/structured_tests/parse_regex.toml
new file mode 100644
index 0000000..8c9ea5f
--- /dev/null
+++ b/tests/structured_tests/parse_regex.toml
@@ -0,0 +1,29 @@
+query = 'db-1 response | parse regex "in (?P<duration>\d+)ms"'
+input = """
+INFO Server db-1 loaded response in 500ms
+INFO Server db-2 loaded response in 394ms
+INFO Server db-1 loaded request in 394ms
+WARN Server failed to load response
+INFO Server db-1 loaded response in 100ms
+INFO Server db-1 loaded response in 102ms
+INFO Server db-1 loaded response in 102ms
+INFO Server db-3 loaded response in 103ms
+INFO Server db-1 loaded response in 100ms
+INFO Server db-3 loaded response in 109ms
+INFO Server db-1 loaded response in 100ms
+INFO Server db-3 loaded response in 104ms
+INFO Server db-1 loaded response in 100ms
+INFO Server db-3 loaded response in 100ms
+INFO Server db-1 loaded response in 122ms
+INFO Server db-2 loaded response in 119ms
+"""
+output = """
+[duration=500]
+[duration=100]
+[duration=102]
+[duration=102]
+[duration=100]
+[duration=100]
+[duration=100]
+[duration=122]
+"""