Repository: rcoh/angle-grinder (language: rust)

Issue:
Support Apache Combined Log format
Hey this tool is really neat! The addition of the logfmt operator made me think of another type of log file that's in common use which people would likely use `agrinder` against- Apache's Combined log format. Many other web servers, including nginx and caddy, are able to output in combined log format, so adding a parser for this format would immediately make this program useful for a variety of web server logs. Since the fields are well-defined and structured, it shouldn't be too difficult to break them up into a format that's useful for angle-grinder. 


Discussion / hints:
interesting. For formats like this that are parsable with the existing `parser` operator I'm considering having some why of having a "parser library" so that you could write parsers like this without needing to write any Rust.

It would be a folder of files, each containing an angle-grinder pipeline. There would be some builtin formats, but a local folder could also be used.

```
# parsers/apache
# 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
parse "* - * [*] "* * *" * *" as ip, name, ts, method, url, protocol, status, cl
```

Thoughts?
@rcoh, that sounds like a great idea! Having a separate method to create a parser without writing Rust would make it easier for folks like me to contribute to the project in a meaningful technical way. Plus, people could try out other formats locally before submitting them upstream. 

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 72851d0..396e5ed 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,6 +24,7 @@ dependencies = [
  "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "im 13.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "include_dir 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "logfmt 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -486,6 +487,11 @@ dependencies = [
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "glob"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "globset"
 version = "0.4.2"
@@ -662,6 +668,28 @@ dependencies = [
  "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "include_dir"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "include_dir_impl 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro-hack 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "include_dir_impl"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro-hack 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.0.2"
@@ -1093,6 +1121,19 @@ name = "podio"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "proc-macro-hack"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro-hack-impl 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "proc-macro-hack-impl"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "proc-macro2"
 version = "0.4.24"
@@ -1531,6 +1572,16 @@ dependencies = [
  "syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "syn"
+version = "0.14.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "syn"
 version = "0.15.23"
@@ -2087,6 +2138,7 @@ dependencies = [
 "checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b"
 "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
 "checksum getopts 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0a7292d30132fb5424b354f5dc02512a86e4c516fe544bb7a25e7f266951b797"
+"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
 "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865"
 "checksum globwalk 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce5d04da8cf35b507b2cbec92bbf2d5085292d07cd87637994fd437fe1617bbb"
 "checksum h2 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "1ac030ae20dee464c5d0f36544d8b914a6bc606da44a57e052d2b0f5dae129e0"
@@ -2101,6 +2153,8 @@ dependencies = [
 "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
 "checksum ignore 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a7927f029a610b866d545c488dd1fc403c4fed1cb4aea2ad568eaa9fc79cd9"
 "checksum im 13.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8db49f8bc08d5cc4e2bb0f7d25a6d1db2c79bc6f7d7c86c96c657eb3d214125f"
+"checksum include_dir 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f41a8bee1894b3fb755d8f09ccd764650476358197a0582555f698fe84b0ae93"
+"checksum include_dir_impl 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4b029199aef0fb9921fdc5623843197e6f4a035774523817599a9f55e4bf3b"
 "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d"
 "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
 "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358"
@@ -2152,6 +2206,8 @@ dependencies = [
 "checksum phf_shared 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b539898d22d4273ded07f64a05737649dc69095d92cb87c7097ec68e3f150b93"
 "checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c"
 "checksum podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd"
+"checksum proc-macro-hack 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "463bf29e7f11344e58c9e01f171470ab15c925c6822ad75028cc1c0e1d1eb63b"
+"checksum proc-macro-hack-impl 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "38c47dcb1594802de8c02f3b899e2018c78291168a22c281be21ea0fb4796842"
 "checksum proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)" = "77619697826f31a02ae974457af0b29b723e5619e113e9397b8b82c6bd253f09"
 "checksum pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eef52fac62d0ea7b9b4dc7da092aa64ea7ec3d90af6679422d3d7e0e14b6ee15"
 "checksum quantiles 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c10fa813fb26fb6c321a6f3085b5ade4cb4730d08d0b9e70a3759136940957f2"
@@ -2203,6 +2259,7 @@ dependencies = [
 "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 "checksum structopt 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "670ad348dc73012fcf78c71f06f9d942232cdd4c859d4b6975e27836c3efc0c3"
 "checksum structopt-derive 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "ef98172b1a00b0bec738508d3726540edcbd186d50dfd326f2b1febbb3559f04"
+"checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741"
 "checksum syn 0.15.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9545a6a093a3f0bd59adb472700acc08cad3776f860f16a897dfce8c88721cbc"
 "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015"
 "checksum tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "a303ba60a099fcd2aaa646b14d2724591a96a75283e4b7ed3d1a1658909d9ae2"
diff --git a/Cargo.toml b/Cargo.toml
index 51c368d..844661a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,11 +40,12 @@ lazy_static = "1.2.0"
 im = "13.0.0"
 logfmt = "0.0.2"
 strfmt = "0.1.6"
+include_dir = "0.2.1"
+toml = "0.4.10"
+serde = "1.0.82"
+serde_derive = "1.0.82"
 
 [dev-dependencies]
 assert_cli = "0.6.3"
 maplit = "1.0.1"
-toml = "0.4.10"
-serde = "1.0.82"
-serde_derive = "1.0.82"
 pulldown-cmark = "0.2.0"
diff --git a/aliases/apache.toml b/aliases/apache.toml
new file mode 100644
index 0000000..3207ea3
--- /dev/null
+++ b/aliases/apache.toml
@@ -0,0 +1,4 @@
+keyword = "apache"
+template = """
+parse "* - * [*] \\"* * *\\" * *" as ip, name, timestamp, method, url, protocol, status, contentlength
+"""
diff --git a/aliases/multi-operator.toml b/aliases/multi-operator.toml
new file mode 100644
index 0000000..7f2c0a5
--- /dev/null
+++ b/aliases/multi-operator.toml
@@ -0,0 +1,4 @@
+keyword = "testmultioperator"
+template = """
+json | count(abc)
+"""
diff --git a/src/alias.rs b/src/alias.rs
new file mode 100644
index 0000000..44ef236
--- /dev/null
+++ b/src/alias.rs
@@ -0,0 +1,48 @@
+//! Instructions on adding a new alias:
+//! 1. Create a new file for the alias in `aliases`.
+//!     1a. The filename is the string to be replaced.
+//!     1b. The string inside the file is the replacement.
+//! 2. Create a new test config inside `tests/structured_tests/aliases`.
+//! 3. Add the test config to the `test_aliases()` test.
+use lazy_static::lazy_static;
+
+use include_dir::Dir;
+
+const ALIASES_DIR: Dir = include_dir!("aliases");
+
+lazy_static! {
+    pub static ref LOADED_ALIASES: Vec<AliasConfig> = ALIASES_DIR
+        .files()
+        .iter()
+        .map(|file| {
+            toml::from_str(file.contents_utf8().expect("load string")).expect("toml valid")
+        })
+        .collect();
+    pub static ref LOADED_KEYWORDS: Vec<&'static str> =
+        LOADED_ALIASES.iter().map(|a| a.keyword.as_str()).collect();
+}
+
+#[derive(Debug, Deserialize, PartialEq)]
+pub struct AliasConfig {
+    keyword: String,
+    template: String,
+}
+
+impl AliasConfig {
+    pub fn matching_string(s: String) -> Result<&'static AliasConfig, ()> {
+        for alias in LOADED_ALIASES.iter() {
+            if alias.keyword != s {
+                continue;
+            }
+
+            return Ok(alias);
+        }
+
+        Err(())
+    }
+
+    /// Render the alias as a string that should parse into a valid operator.
+    pub fn render(&self) -> String {
+        self.template.clone()
+    }
+}
diff --git a/src/lang.rs b/src/lang.rs
index 1ff1d56..5b6f504 100644
--- a/src/lang.rs
+++ b/src/lang.rs
@@ -1,3 +1,4 @@
+use crate::alias::{self, AliasConfig};
 use crate::data;
 use crate::errors::SyntaxErrors;
 use lazy_static::lazy_static;
@@ -37,10 +38,10 @@ macro_rules! with_pos {
 }
 
 /// Dynamic version of `alt` that takes a slice of strings
-fn alternative<T>(input: T, alternatives: &[&'static str]) -> IResult<T, T>
+fn alternative<'a, T>(input: T, alternatives: &[&'a str]) -> IResult<T, T>
 where
     T: InputTake,
-    T: Compare<&'static str>,
+    T: Compare<&'a str>,
     T: InputLength,
     T: AtEof,
     T: Clone,
@@ -48,7 +49,7 @@ where
     let mut last_err = None;
     for alternative in alternatives {
         let inp = input.clone();
-        match tag!(inp, &**alternative) {
+        match tag!(inp, alternative) {
             done @ Ok(..) => return done,
             err @ Err(..) => last_err = Some(err), // continue
         }
@@ -72,8 +73,14 @@ pub const VALID_INLINE: &'static [&str] = &[
 ];
 
 lazy_static! {
-    pub static ref VALID_OPERATORS: Vec<&'static str> =
-        { [VALID_INLINE, VALID_AGGREGATES].concat() };
+    pub static ref VALID_OPERATORS: Vec<&'static str> = {
+        [
+            VALID_INLINE,
+            VALID_AGGREGATES,
+            alias::LOADED_KEYWORDS.as_slice(),
+        ]
+        .concat()
+    };
 }
 
 pub const RESERVED_FILTER_WORDS: &'static [&str] = &["AND", "OR", "NOT"];
@@ -215,6 +222,7 @@ impl Search {
 
 #[derive(Debug, PartialEq)]
 pub enum Operator {
+    RenderedAlias(Positioned<String>),
     Inline(Positioned<InlineOperator>),
     MultiAggregate(MultiAggregateOperator),
     Sort(SortOperator),
@@ -512,7 +520,7 @@ named!(sourced_expr<Span, (String, Expr)>, ws!(
         )
 )));
 
-named_args!(did_you_mean<'a>(choices: &[&'static str], err: SyntaxErrors)<Span<'a>, Span<'a>>,
+named_args!(did_you_mean<'a>(choices: &[&'a str], err: SyntaxErrors)<Span<'a>, Span<'a>>,
         preceded!(space0,
             return_error!(SyntaxErrors::StartOfError.into(),
                 alt_complete!(
@@ -636,6 +644,20 @@ named!(p_nn<Span, Positioned<AggregateFunction>>, ws!(
     ))
 ));
 
+fn string_from_located_span(
+    span: LocatedSpan<impl nom::AsBytes>,
+) -> Result<String, std::string::FromUtf8Error> {
+    String::from_utf8(span.fragment.as_bytes().to_vec())
+}
+
+named!(alias<Span, Operator>, map!(
+    with_pos!(ws!(do_parse!(
+        config: map_res!(map_res!(nom::alpha, string_from_located_span), AliasConfig::matching_string) >>
+        (config.render())
+    ))),
+    Operator::RenderedAlias
+));
+
 named!(inline_operator<Span, Operator>,
     map!(alt_complete!(parse | json | logfmt | fields | whre | limit | total | split), Operator::Inline)
 );
@@ -654,7 +676,7 @@ named!(aggregate_function<Span, Positioned<AggregateFunction>>, do_parse!(
 
 named!(operator<Span, Operator>, do_parse!(
     peek!(did_you_mean_operator) >>
-    res: alt_complete!(inline_operator | sort | multi_aggregate_operator) >> (res)
+    res: alt_complete!(inline_operator | sort | alias | multi_aggregate_operator) >> (res)
 ));
 
 // count by x,y
@@ -816,13 +838,15 @@ named!(prelexed_query<Span, Search>, ws!(do_parse!(
 
 named!(pub query<Span, Query, SyntaxErrors>, fix_error!(SyntaxErrors, exact!(ws!(do_parse!(
     filter: prelexed_query >>
-    operators: ws!(opt!(preceded!(tag!("|"), ws!(separated_nonempty_list!(tag!("|"), operator))))) >>
+    operators: ws!(opt!(preceded!(tag!("|"), operator_list))) >>
     (Query{
         search: filter,
         operators: operators.unwrap_or_default()
     }))
 ))));
 
+named!(pub operator_list<Span, Vec<Operator>>, ws!(separated_nonempty_list!(tag!("|"), operator)));
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/lib.rs b/src/lib.rs
index 6fe3779..3cadee4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,13 +1,20 @@
 #[macro_use]
 extern crate failure;
+#[macro_use]
+extern crate include_dir;
+#[macro_use]
+extern crate serde_derive;
 extern crate atty;
 extern crate nom_locate;
 extern crate num_derive;
 extern crate num_traits;
+extern crate serde;
+extern crate toml;
 
 extern crate annotate_snippets;
 extern crate crossbeam_channel;
 
+mod alias;
 mod data;
 mod errors;
 mod filter;
@@ -26,6 +33,8 @@ pub mod pipeline {
     use crate::typecheck::{TypeCheck, TypeError};
     use crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender};
     use failure::Error;
+    use nom::types::CompleteStr;
+    use std::collections::VecDeque;
     use std::io::BufRead;
     use std::thread;
     use std::time::Duration;
@@ -114,10 +123,28 @@ pub mod pipeline {
             let mut in_agg = false;
             let mut pre_agg: Vec<Box<dyn operator::UnaryPreAggOperator>> = Vec::new();
             let mut post_agg: Vec<Box<dyn operator::AggregateOperator>> = Vec::new();
-            let mut op_iter = query.operators.into_iter().peekable();
+            let mut op_deque = query.operators.into_iter().collect::<VecDeque<_>>();
             let mut has_errors = false;
-            while let Some(op) = op_iter.next() {
+            while let Some(op) = op_deque.pop_front() {
                 match op {
+                    Operator::RenderedAlias(rendered_alias) => {
+                        // TODO: create a new QueryContainer here and parse the templated string
+                        // -> expecting only Inline operators?
+                        // let inner_query = QueryContainer::new(rendered_alias.value, ???);
+                        let (_span, operators) =
+                            match operator_list(Span::new(CompleteStr(&rendered_alias.value))) {
+                                Err(_err) => bail!(
+                                    "Failed to parse rendered alias: `{}` as operator_list",
+                                    rendered_alias.value,
+                                ),
+                                Ok(v) => v,
+                            };
+
+                        // Insert operators (in-order) to front of operator stack
+                        for new_op in operators.into_iter().rev() {
+                            op_deque.push_front(new_op);
+                        }
+                    }
                     Operator::Inline(inline_op) => {
                         let op_builder = inline_op.type_check(pipeline)?;
 
@@ -133,7 +160,7 @@ pub mod pipeline {
                         if let Ok(op) = Pipeline::convert_multi_agg(agg_op, pipeline) {
                             post_agg.push(op);
 
-                            let needs_sort = match op_iter.peek() {
+                            let needs_sort = match op_deque.front() {
                                 Some(Operator::Inline(Positioned {
                                     value: InlineOperator::Limit { .. },
                                     ..


Test patch:
diff --git a/tests/integration.rs b/tests/integration.rs
index 6052615..573a5e0 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -166,6 +166,15 @@ mod integration {
         structured_test(include_str!("structured_tests/split_10.toml"));
     }
 
+    #[test]
+    fn test_aliases() {
+        structured_test(include_str!("structured_tests/aliases/apache.toml"));
+        structured_test(include_str!("structured_tests/aliases/multi-operator.toml"));
+        structured_test(include_str!(
+            "structured_tests/aliases/alias_with_failure.toml"
+        ));
+    }
+
     #[test]
     fn no_args() {
         assert_cli::Assert::main_binary()
diff --git a/tests/structured_tests/aliases/alias_with_failure.toml b/tests/structured_tests/aliases/alias_with_failure.toml
new file mode 100644
index 0000000..0002308
--- /dev/null
+++ b/tests/structured_tests/aliases/alias_with_failure.toml
@@ -0,0 +1,13 @@
+query = """* | apache | foo"""
+input = ""
+output = ""
+error = """
+error: Expected an operator
+  |
+1 | * | apache | foo
+  |              ^^^
+  |
+  = help: foo is not a valid operator
+Error: Failed to parse query
+"""
+succeeds = false
diff --git a/tests/structured_tests/aliases/apache.toml b/tests/structured_tests/aliases/apache.toml
new file mode 100644
index 0000000..c3748bb
--- /dev/null
+++ b/tests/structured_tests/aliases/apache.toml
@@ -0,0 +1,7 @@
+query = """* | apache"""
+input = """
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
+"""
+output = """
+[contentlength=2326]                 [ip=127.0.0.1]        [method=GET]           [name=frank]        [protocol=HTTP/1.0]        [status=200]           [timestamp=10/Oct/2000:13:55:36 -0700]        [url=/apache_pb.gif]
+"""
diff --git a/tests/structured_tests/aliases/multi-operator.toml b/tests/structured_tests/aliases/multi-operator.toml
new file mode 100644
index 0000000..931edc8
--- /dev/null
+++ b/tests/structured_tests/aliases/multi-operator.toml
@@ -0,0 +1,9 @@
+query = """* | testmultioperator"""
+input = """
+{ "abc": 5, "xyz": "hello" }
+"""
+output = """
+_count
+--------------
+1
+"""