Repository: rcoh/angle-grinder (language: rust)

Issue:
More control over human-formatted output
I'd like to use agrind to transform structured log output into something that's easy to read while tailing some logs

Currently agrind's human-formatted output looks something like
```
[id=ConsumerFetcherManager-1382721708341][level=info][module=kafka.consumer.ConsumerFetcherManager][msg=Stopping all fetchers][tag=stopping_fetchers]
```

I'd like more control over the output formatting so I could turn it into something like
```
info | Stopping all fetchers          tag=stopping_fetchers
```

That seems possible by exposing a `printf`-style format operator. The above would be something like
```
* | logfmt | printf "%s | %30s tag=%s" level, msg, tag
```


Discussion / hints:
This seems related to #71 but with a different goal.

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 8cb06ea..46aec8a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,6 +41,7 @@ dependencies = [
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strfmt 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "structopt 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "terminal_size 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1490,6 +1491,11 @@ name = "stable_deref_trait"
 version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "strfmt"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "string"
 version = "0.1.2"
@@ -2191,6 +2197,7 @@ dependencies = [
 "checksum slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5f9776d6b986f77b35c6cf846c11ad986ff128fe0b2b63a3628e3755e8d3102d"
 "checksum smallvec 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b73ea3738b47563803ef814925e69be00799a8c07420be8b996f8e98fb2336db"
 "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8"
+"checksum strfmt 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b278b244ef7aa5852b277f52dd0c6cac3a109919e1f6d699adde63251227a30f"
 "checksum string 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98998cced76115b1da46f63388b909d118a37ae0be0f82ad35773d4a4bc9d18d"
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
 "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
diff --git a/Cargo.toml b/Cargo.toml
index 832844b..a716bb5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,6 +39,7 @@ atty = "0.2.0"
 lazy_static = "1.2.0"
 im = "13.0.0"
 logfmt = { git = "https://github.com/brandur/logfmt", tag = "v0.0.2" }
+strfmt = "0.1.6"
 
 [dev-dependencies]
 assert_cli = "0.6.3"
diff --git a/README.md b/README.md
index ac32061..26205f3 100644
--- a/README.md
+++ b/README.md
@@ -297,6 +297,13 @@ tail -f live_pcap | agrind '* | parse "* > *:" as src, dest | parse "length *" a
 [dest=111.221.29.254.https]        [length=310]      [src=21:50:18.458527 IP 10.0.2.243.47152]
 ```
 
+Alternate rendering formats can be provided with the `--format` flag. This flag uses the formatting syntax defined in https://doc.rust-lang.org/std/fmt/#syntax. For example
+```
+tail -f live_pcap | agrind --format '{src} => {dst} | length={length}' '* | parse "* > *:" as src, dest | parse "length *" as length'
+21:50:18.458331 IP 10.0.2.243.47152 => 111.221.29.254.https | length=0
+21:50:18.458527 IP 10.0.2.243.47152 => 111.221.29.254.https | length=310
+```
+
 Aggregate data is written to the terminal and will live-update until the stream ends:
 ```noformat
 k2                  avg         
diff --git a/src/bin/agrind.rs b/src/bin/agrind.rs
index 524094a..f112a47 100644
--- a/src/bin/agrind.rs
+++ b/src/bin/agrind.rs
@@ -35,6 +35,11 @@ struct Cli {
     /// Optionally reads from a file instead of Stdin
     #[structopt(long = "file", short = "f")]
     file: Option<String>,
+
+    /// Provide a Rust std::fmt string to format output
+    #[structopt(long = "format", short = "m")]
+    format: Option<String>,
+
     #[structopt(flatten)]
     verbosity: Verbosity,
 }
@@ -73,7 +78,7 @@ fn main() -> CliResult {
         }),
     );
     args.verbosity.setup_env_logger("agrind")?;
-    let pipeline = Pipeline::new(&query)?;
+    let pipeline = Pipeline::new(&query, args.format)?;
     match args.file {
         Some(file_name) => {
             let f = File::open(file_name)?;
diff --git a/src/lib.rs b/src/lib.rs
index 51d538f..ebdae2e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -110,7 +110,7 @@ pub mod pipeline {
             }
         }
 
-        pub fn new(pipeline: &QueryContainer) -> Result<Self, Error> {
+        pub fn new(pipeline: &QueryContainer, format: Option<String>) -> Result<Self, Error> {
             let parsed = pipeline.parse().map_err(|_pos| CompileError::Parse);
             let query = parsed?;
             let filters = Pipeline::convert_filter(query.search);
@@ -166,6 +166,7 @@ pub mod pipeline {
                         floating_points: 2,
                         min_buffer: 4,
                         max_buffer: 8,
+                        format: format,
                     },
                     Duration::from_millis(50),
                 ),
diff --git a/src/render.rs b/src/render.rs
index e72b36f..c8d9ff1 100644
--- a/src/render.rs
+++ b/src/render.rs
@@ -4,6 +4,9 @@ use std;
 use std::collections::HashMap;
 use std::io::{stdout, Write};
 
+extern crate strfmt;
+use strfmt::strfmt;
+
 extern crate terminal_size;
 
 use self::terminal_size::{terminal_size, Height, Width};
@@ -13,6 +16,7 @@ pub struct RenderConfig {
     pub floating_points: usize,
     pub min_buffer: usize,
     pub max_buffer: usize,
+    pub format: Option<String>,
 }
 
 struct TerminalSize {
@@ -104,7 +108,7 @@ impl PrettyPrinter {
         }
     }
 
-    fn format_record(&mut self, record: &data::Record) -> String {
+    fn format_record_as_columns(&mut self, record: &data::Record) -> String {
         let new_column_widths = self.compute_column_widths(&(record.data));
         self.column_widths.extend(new_column_widths);
         let new_columns = self.new_columns(&(record.data));
@@ -148,6 +152,17 @@ impl PrettyPrinter {
         strs.join("").trim().to_string()
     }
 
+    fn format_record_as_format(&self, format: &String, record: &data::Record) -> String {
+        strfmt(format, &record.data).unwrap()
+    }
+
+    fn format_record(&mut self, record: &data::Record) -> String {
+        match self.render_config.format {
+            Some(ref format) => self.format_record_as_format(format, record),
+            None => self.format_record_as_columns(record),
+        }
+    }
+
     fn max_width(&self) -> u16 {
         match self.term_size {
             None => 240,
@@ -328,6 +343,7 @@ mod tests {
                 floating_points: 2,
                 min_buffer: 1,
                 max_buffer: 4,
+                format: None,
             },
             None,
         );
@@ -344,6 +360,7 @@ mod tests {
                 floating_points: 2,
                 min_buffer: 1,
                 max_buffer: 4,
+                format: None,
             },
             None,
         );
@@ -370,6 +387,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn pretty_print_record_formatted() {
+        let rec = Record::new(r#"{"k1": 5, "k2": 5.5000001, "k3": "str"}"#);
+        let parser = ParseJson::new(None);
+        let rec = parser.process(rec).unwrap().unwrap();
+        let mut pp = PrettyPrinter::new(
+            RenderConfig {
+                floating_points: 2,
+                min_buffer: 1,
+                max_buffer: 4,
+                format: Some("{k1:>3} k2={k2:<10.3} k3[{k3}]".to_string()),
+            },
+            None,
+        );
+        assert_eq!(pp.format_record(&rec), "  5 k2=5.5        k3[str]");
+        let rec = Record::new(r#"{"k1": 955, "k2": 5.5000001, "k3": "str3"}"#);
+        let parser = ParseJson::new(None);
+        let rec = parser.process(rec).unwrap().unwrap();
+        assert_eq!(pp.format_record(&rec), "955 k2=5.5        k3[str3]");
+        let rec = Record::new(
+            r#"{"k1": "here is a amuch longer stsring", "k2": 5.5000001, "k3": "str3"}"#,
+        );
+        let parser = ParseJson::new(None);
+        let rec = parser.process(rec).unwrap().unwrap();
+        assert_eq!(
+            pp.format_record(&rec),
+            "here is a amuch longer stsring k2=5.5        k3[str3]"
+        );
+    }
+
     #[test]
     fn pretty_print_record_too_long() {
         let rec = Record::new(r#"{"k1": 5, "k2": 5.5000001, "k3": "str"}"#);
@@ -380,6 +427,7 @@ mod tests {
                 floating_points: 2,
                 min_buffer: 1,
                 max_buffer: 4,
+                format: None,
             },
             Some(TerminalSize {
                 width: 10,
@@ -417,6 +465,7 @@ mod tests {
                 floating_points: 2,
                 min_buffer: 2,
                 max_buffer: 4,
+                format: None,
             },
             Some(TerminalSize {
                 width: 100,
@@ -465,6 +514,7 @@ mod tests {
                 floating_points: 2,
                 min_buffer: 2,
                 max_buffer: 4,
+                format: None,
             },
             Some(TerminalSize {
                 width: max_width as u16,


Test patch:
diff --git a/tests/integration.rs b/tests/integration.rs
index 7c5b969..45f2e16 100644
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -311,9 +311,28 @@ None         1")
             .unwrap();
     }
 
+    #[test]
+    fn custom_format() {
+        assert_cli::Assert::main_binary()
+            .with_args(&[
+                "* | logfmt",
+                "--format",
+                "{level} | {msg:<30} module={module}",
+                "--file",
+                "test_files/test_logfmt.log",
+            ])
+            .stdout()
+            .is(
+                "info | Stopping all fetchers          module=kafka.consumer.ConsumerFetcherManager
+info | Starting all fetchers          module=kafka.consumer.ConsumerFetcherManager
+warn | Fetcher failed to start        module=kafka.consumer.ConsumerFetcherManager",
+            )
+            .unwrap();
+    }
+
     fn ensure_parses(query: &str) {
         let query_container = QueryContainer::new(query.to_string(), Box::new(EmptyErrorReporter));
-        Pipeline::new(&query_container).expect(&format!(
+        Pipeline::new(&query_container, None).expect(&format!(
             "Query: `{}` from the README should have parsed",
             query
         ));