Repository: rcoh/angle-grinder (language: rust)

Issue:
Support parsing tabs
It would be useful to be able to parse tabs.

For example, I have logs like:
```
 [WARNING]2020-01-01T00:00:00.000Z        1e4cd606-a550-4d41-b013-f61a4a3d959d      init_attendance no students user='kennimrod'
/aws/lambda/function-4TPZE69CJ8X7 2020/01/01/[$LATEST]e75d9ab689c4457da0a79316cf624c19 [WARNING]	2020-01-01T00:00:00.000Z	1e4cd106-a551-4d41-b313-f61a4a3d059d	init_attendance no students user='user'
```

Note that there are some tabs in between some of the parts.

I tried `ag '* | parse "* * [*]\t.*" as lambda_name, run_instance, log_level, datetime' < log` but it does not work, seemed like it does not recognize `\t`.

One solution in to process it with `sed` or `sd` to substitute the tabs first. For example, `sd \t , < log | ag '* | parse "* * [*],*,*,*" as lambda_name, run_instance, log_level, datetime, request_id, message`.


Discussion / hints:
Did you try putting a literal tab in the pattern? I agree this is needed either way though
I didn't try that because it is not easy to put a literal tab here, I forgot to think about using edit mode to add a literal tab.

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index e1dc3e8..7e6cd61 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12,6 +12,7 @@ dependencies = [
  "annotate-snippets 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "assert_cli 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cargo-husky 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "criterion 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -39,6 +40,7 @@ dependencies = [
  "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "terminal_size 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "test-generator 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -208,6 +210,11 @@ dependencies = [
  "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "cargo-husky"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "cast"
 version = "0.2.3"
@@ -608,6 +615,11 @@ name = "glob"
 version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "glob"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "globset"
 version = "0.4.4"
@@ -1860,6 +1872,17 @@ dependencies = [
  "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "test-generator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -2359,6 +2382,7 @@ dependencies = [
 "checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b"
 "checksum bzip2-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6584aa36f5ad4c9247f5323b0a42f37802b37a836f0ad87084d7a33961abe25f"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
+"checksum cargo-husky 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"
 "checksum cast 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0"
 "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
 "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
@@ -2405,6 +2429,7 @@ dependencies = [
 "checksum getopts 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
 "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
+"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 "checksum globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "925aa2cac82d8834e2b2a4415b6f6879757fb5c0928fc445ae76461a12eed8f2"
 "checksum globwalk 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce5d04da8cf35b507b2cbec92bbf2d5085292d07cd87637994fd437fe1617bbb"
 "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462"
@@ -2542,6 +2567,7 @@ dependencies = [
 "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
 "checksum terminal_size 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b"
 "checksum termion 1.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c22cec9d8978d906be5ac94bceb5a010d885c626c4c8855721a4dbd20e3ac905"
+"checksum test-generator 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea97be90349ab3574f6e74d1566e1c5dd3a4bc74b89f4af4cc10ca010af103c0"
 "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
 "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
diff --git a/Cargo.toml b/Cargo.toml
index f57734e..042af78 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -50,6 +50,11 @@ assert_cli = "0.6.3"
 pulldown-cmark = "0.2.0"
 criterion = "0.3"
 maplit = "1.0.1"
+test-generator = "0.3.0"
+[dev-dependencies.cargo-husky]
+version = "1"
+default-features = true
+features = ["precommit-hook", "run-cargo-fmt"]
 
 [[bench]]
 name = "e2e"
diff --git a/README.md b/README.md
index bd5e92d..daba4f6 100644
--- a/README.md
+++ b/README.md
@@ -169,7 +169,8 @@ Other examples:
 
 ##### Parse
 `parse "* pattern * otherpattern *" [from field] as a,b,c [nodrop]`: Parse text that matches the pattern into variables. Lines that don't match the pattern will be dropped unless `nodrop` is specified. `*` is equivalent to regular expression `.*` and is greedy.
-By default, `parse` operates on the raw text of the message. With `from field_name`, parse will instead process input from a specific column.
+By default, `parse` operates on the raw text of the message. With `from field_name`, parse will instead process input from a specific column. Any whitespace in the parse
+expression will match _any_ whitespace character in the input text (eg. a literal tab).
 
 *Examples*:
 ```agrind
diff --git a/src/lang.rs b/src/lang.rs
index 0265a8a..bddd5d7 100644
--- a/src/lang.rs
+++ b/src/lang.rs
@@ -188,7 +188,7 @@ impl Keyword {
 
     /// Convert this keyword to a `regex::Regex` object.
     pub fn to_regex(&self) -> regex::Regex {
-        let mut regex_str = regex::escape(&self.0.replace("\\\"", "\""));
+        let mut regex_str = regex::escape(&self.0.replace("\\\"", "\"")).replace(" ", "\\s");
 
         regex_str.insert_str(0, "(?i)");
         if self.1 == KeywordType::WILDCARD {
diff --git a/src/operator.rs b/src/operator.rs
index 9b0b8a3..95fe14b 100644
--- a/src/operator.rs
+++ b/src/operator.rs
@@ -1356,6 +1356,21 @@ mod tests {
         assert_eq!(rec.data.get("ip").unwrap(), &Value::None);
     }
 
+    #[test]
+    fn parse_all_whitespace() {
+        let rec = Record::new("abcd\t1234\t4567");
+        let parser = Parse::new(
+            lang::Keyword::new_wildcard("abcd * *".to_string()).to_regex(),
+            vec!["num_1".to_string(), "num_2".to_string()],
+            None,
+            ParseOptions {
+                drop_nonmatching: false,
+            },
+        );
+        let parsed = parser.process(rec).unwrap().unwrap();
+        assert_eq!(parsed.data.get("num_1"), Some(&Value::Int(1234)))
+    }
+
     #[test]
     fn parse_nodrop_preserve_existing() {
         let rec = Record::new("abcd 1234").put("ip", Value::Str("127.0.0.1".to_string()));


Test patch:
diff --git a/tests/integration.rs b/tests/integration.rs
index 6320fdd..22d2b5b 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -1,9 +1,12 @@
+#![cfg(test)]
 extern crate ag;
 extern crate assert_cli;
 extern crate pulldown_cmark;
+extern crate test_generator;
 extern crate toml;
 
 use serde_derive::Deserialize;
+use test_generator::test_resources;
 
 mod code_blocks;
 
@@ -23,6 +26,7 @@ mod integration {
     use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer};
     use assert_cli;
     use std::borrow::Borrow;
+    use std::fs;
     use std::io::stdout;
     use toml;
 
@@ -30,8 +34,10 @@ mod integration {
 
     impl ErrorReporter for EmptyErrorReporter {}
 
-    fn structured_test(s: &str) {
-        let conf: TestDefinition = toml::from_str(s).unwrap();
+    #[test_resources("tests/structured_tests/*.toml")]
+    fn structured_test(path: &str) {
+        let contents = fs::read_to_string(path).unwrap();
+        let conf: TestDefinition = toml::from_str(&contents).unwrap();
         let out: &str = conf.output.borrow();
         let err = conf.error.unwrap_or("".to_string());
         let env = assert_cli::Environment::inherit().insert("RUST_BACKTRACE", "0");
@@ -52,130 +58,6 @@ mod integration {
         asserter.unwrap();
     }
 
-    #[test]
-    fn count_distinct_operator() {
-        structured_test(include_str!("structured_tests/count_distinct.toml"));
-        structured_test(include_str!("structured_tests/count_distinct_error.toml"));
-    }
-
-    #[test]
-    fn long_aggregate_values() {
-        structured_test(include_str!("structured_tests/longlines.toml"));
-    }
-
-    #[test]
-    fn parse_operator() {
-        structured_test(include_str!(
-            "structured_tests/parse_error_unterminated.toml"
-        ));
-        structured_test(include_str!(
-            "structured_tests/parse_error_unterminated_sq.toml"
-        ));
-        structured_test(include_str!("structured_tests/parse_drop.toml"));
-        structured_test(include_str!("structured_tests/parse_nodrop.toml"));
-    }
-
-    #[test]
-    fn logfmt_operator() {
-        structured_test(include_str!("structured_tests/logfmt.toml"));
-    }
-
-    #[test]
-    fn sum_operator() {
-        structured_test(include_str!("structured_tests/sum.toml"));
-    }
-
-    #[test]
-    fn min_max_operators() {
-        structured_test(include_str!("structured_tests/min_max.toml"));
-        structured_test(include_str!("structured_tests/min_max_none.toml"));
-    }
-
-    #[test]
-    fn where_operator() {
-        structured_test(include_str!("structured_tests/where-1.toml"));
-        structured_test(include_str!("structured_tests/where-2.toml"));
-        structured_test(include_str!("structured_tests/where-3.toml"));
-        structured_test(include_str!("structured_tests/where-4.toml"));
-        structured_test(include_str!("structured_tests/where-5.toml"));
-        structured_test(include_str!("structured_tests/where-6.toml"));
-        structured_test(include_str!("structured_tests/where-7.toml"));
-        structured_test(include_str!("structured_tests/where-8.toml"));
-    }
-
-    #[test]
-    fn filters() {
-        structured_test(include_str!("structured_tests/filters.toml"));
-    }
-
-    #[test]
-    fn sort_order() {
-        structured_test(include_str!("structured_tests/sort_order.toml"));
-    }
-
-    #[test]
-    fn total() {
-        structured_test(include_str!("structured_tests/total.toml"));
-        structured_test(include_str!("structured_tests/total_agg.toml"));
-    }
-
-    #[test]
-    fn fields_after_agg_bug() {
-        structured_test(include_str!("structured_tests/fields_after_agg.toml"));
-    }
-
-    #[test]
-    fn limit() {
-        structured_test(include_str!("structured_tests/limit.toml"));
-        structured_test(include_str!("structured_tests/limit_tail.toml"));
-        structured_test(include_str!("structured_tests/limit_agg.toml"));
-        structured_test(include_str!("structured_tests/limit_agg_tail.toml"));
-    }
-
-    #[test]
-    fn suggest_alternatives() {
-        structured_test(include_str!("structured_tests/limit_error.toml"));
-        structured_test(include_str!("structured_tests/limit_error_2.toml"));
-        structured_test(include_str!("structured_tests/count_distinct_error_2.toml"));
-        structured_test(include_str!("structured_tests/not_an_agg.toml"));
-        structured_test(include_str!("structured_tests/not_an_agg_2.toml"))
-    }
-
-    #[test]
-    fn test_json_arrays() {
-        structured_test(include_str!("structured_tests/arrays_1.toml"));
-    }
-
-    #[test]
-    fn test_nested_values() {
-        structured_test(include_str!("structured_tests/nested_values_1.toml"));
-        structured_test(include_str!("structured_tests/nested_values_2.toml"));
-        structured_test(include_str!("structured_tests/nested_values_3.toml"));
-    }
-
-    #[test]
-    fn test_split() {
-        structured_test(include_str!("structured_tests/split_1.toml"));
-        structured_test(include_str!("structured_tests/split_2.toml"));
-        structured_test(include_str!("structured_tests/split_3.toml"));
-        structured_test(include_str!("structured_tests/split_4.toml"));
-        structured_test(include_str!("structured_tests/split_5.toml"));
-        structured_test(include_str!("structured_tests/split_6.toml"));
-        structured_test(include_str!("structured_tests/split_7.toml"));
-        structured_test(include_str!("structured_tests/split_8.toml"));
-        structured_test(include_str!("structured_tests/split_9.toml"));
-        structured_test(include_str!("structured_tests/split_10.toml"));
-    }
-
-    #[test]
-    fn test_aliases() {
-        structured_test(include_str!("structured_tests/aliases/apache.toml"));
-        structured_test(include_str!("structured_tests/aliases/multi-operator.toml"));
-        structured_test(include_str!(
-            "structured_tests/aliases/alias_with_failure.toml"
-        ));
-    }
-
     #[test]
     fn no_args() {
         assert_cli::Assert::main_binary()
diff --git a/tests/structured_tests/parse_literal_tab.toml b/tests/structured_tests/parse_literal_tab.toml
new file mode 100644
index 0000000..8d3f60a
--- /dev/null
+++ b/tests/structured_tests/parse_literal_tab.toml
@@ -0,0 +1,10 @@
+query = """* | parse "name=* says=*" as name, words"""
+input = """
+name=Jarrad\tsays=Hi
+name=Ella says=Bye
+name=Jim
+"""
+output = """
+[name=Jarrad]        [words=Hi]
+[name=Ella]          [words=Bye]
+"""