Repository: pest-parser/pest (language: rust)

Issue:
Feature request: export lexical information as json/xml/whatever
I hope there can be a feature that allows us to pretty-print a json/xml/csv/whatever which includes the lexicial information (like token type, pos/line/column), like the AST printer in https://pest.rs but I want an easy way to invoke that locally and get a more easy-to-parse format (I hope it's csv (you can represent nested tree-structure via adding a `depth` attribute in each column), so I don't even need a json parser).

After this, we can make a lot of goodies like some other tools that renders a webpage of code that is syntax-highlighted according to the lexical information.


Discussion / hints:
This would be a pretty easy addition for 2.2. It should come after a feature since we don't want to bring serde as a mandatory dependency.
Implementation sketch: `impl Serialize for Pair && Pairs`. `Pairs` serializes as `{pos: (start, end), pairs: [Pair]}` and `Pair` serializes as `{pos: (start, end), token: Token, inner: (&str | Pairs)}`. It doesn't have to be super performant, so it can do most of the work through adapter types with `#[derive(Serialize)]`. (That said, `Serialize` isn't too hard to implement correctly manually, so a solution that doesn't use adapter types wouldn't be hard, and `Deserialize` is probably not worth the effort (as we'd have to stuff the full string somewhere or rebuild it)).  
I'd like to work on this issue
@Goncalerta, that's nice to hear! Feel free to take a jab at it following @CAD97's sketch.

Fix patch:
diff --git a/pest/Cargo.toml b/pest/Cargo.toml
index 720bcd86..06d5a642 100644
--- a/pest/Cargo.toml
+++ b/pest/Cargo.toml
@@ -11,8 +11,14 @@ categories = ["parsing"]
 license = "MIT/Apache-2.0"
 readme = "_README.md"
 
+[features]
+# Enables the `to_json` function for `Pair` and `Pairs`
+pretty-print = ["serde", "serde_json"]
+
 [dependencies]
 ucd-trie = "0.1.1"
+serde = { version = "1.0.89", optional = true }
+serde_json = { version = "1.0.39", optional = true}
 
 [badges]
 codecov = { repository = "pest-parser/pest" }
diff --git a/pest/src/iterators/pair.rs b/pest/src/iterators/pair.rs
index 554f70e9..8d8f7690 100644
--- a/pest/src/iterators/pair.rs
+++ b/pest/src/iterators/pair.rs
@@ -13,6 +13,9 @@ use std::ptr;
 use std::rc::Rc;
 use std::str;
 
+#[cfg(feature = "pretty-print")]
+use serde::ser::SerializeStruct;
+
 use super::pairs::{self, Pairs};
 use super::queueable_token::QueueableToken;
 use super::tokens::{self, Tokens};
@@ -234,6 +237,13 @@ impl<'i, R: RuleType> Pair<'i, R> {
         tokens::new(self.queue, self.input, self.start, end + 1)
     }
 
+    /// Generates a string that stores the lexical information of `self` in
+    /// a pretty-printed JSON format.
+    #[cfg(feature = "pretty-print")]
+    pub fn to_json(&self) -> String {
+        ::serde_json::to_string_pretty(self).expect("Failed to pretty-print Pair to json.")
+    }
+
     fn pair(&self) -> usize {
         match self.queue[self.start] {
             QueueableToken::Start {
@@ -305,11 +315,68 @@ impl<'i, R: Hash> Hash for Pair<'i, R> {
     }
 }
 
+#[cfg(feature = "pretty-print")]
+impl<'i, R: RuleType> ::serde::Serialize for Pair<'i, R> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: ::serde::Serializer,
+    {
+        let start = self.pos(self.start);
+        let end = self.pos(self.pair());
+        let rule = format!("{:?}", self.as_rule());
+        let inner = self.clone().into_inner();
+
+        let mut ser = serializer.serialize_struct("Pairs", 3)?;
+        ser.serialize_field("pos", &(start, end))?;
+        ser.serialize_field("rule", &rule)?;
+
+        if inner.peek().is_none() {
+            ser.serialize_field("inner", &self.as_str())?;
+        } else {
+            ser.serialize_field("inner", &inner)?;
+        }
+
+        ser.end()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use macros::tests::*;
     use parser::Parser;
 
+    #[test]
+    #[cfg(feature = "pretty-print")]
+    fn test_pretty_print() {
+        let pair = AbcParser::parse(Rule::a, "abcde").unwrap().next().unwrap();
+
+        let expected = r#"{
+  "pos": [
+    0,
+    3
+  ],
+  "rule": "a",
+  "inner": {
+    "pos": [
+      1,
+      2
+    ],
+    "pairs": [
+      {
+        "pos": [
+          1,
+          2
+        ],
+        "rule": "b",
+        "inner": "b"
+      }
+    ]
+  }
+}"#;
+
+        assert_eq!(expected, pair.to_json());
+    }
+
     #[test]
     fn pair_into_inner() {
         let pair = AbcParser::parse(Rule::a, "abcde").unwrap().next().unwrap(); // the tokens a(b())
diff --git a/pest/src/iterators/pairs.rs b/pest/src/iterators/pairs.rs
index 51077f04..42072c0f 100644
--- a/pest/src/iterators/pairs.rs
+++ b/pest/src/iterators/pairs.rs
@@ -13,6 +13,9 @@ use std::ptr;
 use std::rc::Rc;
 use std::str;
 
+#[cfg(feature = "pretty-print")]
+use serde::ser::SerializeStruct;
+
 use super::flat_pairs::{self, FlatPairs};
 use super::pair::{self, Pair};
 use super::queueable_token::QueueableToken;
@@ -178,6 +181,13 @@ impl<'i, R: RuleType> Pairs<'i, R> {
         }
     }
 
+    /// Generates a string that stores the lexical information of `self` in
+    /// a pretty-printed JSON format.
+    #[cfg(feature = "pretty-print")]
+    pub fn to_json(&self) -> String {
+        ::serde_json::to_string_pretty(self).expect("Failed to pretty-print Pairs to json.")
+    }
+
     fn pair(&self) -> usize {
         match self.queue[self.start] {
             QueueableToken::Start {
@@ -268,11 +278,76 @@ impl<'i, R: Hash> Hash for Pairs<'i, R> {
     }
 }
 
+#[cfg(feature = "pretty-print")]
+impl<'i, R: RuleType> ::serde::Serialize for Pairs<'i, R> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: ::serde::Serializer,
+    {
+        let start = self.pos(self.start);
+        let end = self.pos(self.end - 1);
+        let pairs = self.clone().collect::<Vec<_>>();
+
+        let mut ser = serializer.serialize_struct("Pairs", 2)?;
+        ser.serialize_field("pos", &(start, end))?;
+        ser.serialize_field("pairs", &pairs)?;
+        ser.end()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::super::super::macros::tests::*;
     use super::super::super::Parser;
 
+    #[test]
+    #[cfg(feature = "pretty-print")]
+    fn test_pretty_print() {
+        let pairs = AbcParser::parse(Rule::a, "abcde").unwrap();
+
+        let expected = r#"{
+  "pos": [
+    0,
+    5
+  ],
+  "pairs": [
+    {
+      "pos": [
+        0,
+        3
+      ],
+      "rule": "a",
+      "inner": {
+        "pos": [
+          1,
+          2
+        ],
+        "pairs": [
+          {
+            "pos": [
+              1,
+              2
+            ],
+            "rule": "b",
+            "inner": "b"
+          }
+        ]
+      }
+    },
+    {
+      "pos": [
+        4,
+        5
+      ],
+      "rule": "c",
+      "inner": "e"
+    }
+  ]
+}"#;
+
+        assert_eq!(expected, pairs.to_json());
+    }
+
     #[test]
     fn as_str() {
         let pairs = AbcParser::parse(Rule::a, "abcde").unwrap();
diff --git a/pest/src/lib.rs b/pest/src/lib.rs
index 8f2de1b5..c55518c2 100644
--- a/pest/src/lib.rs
+++ b/pest/src/lib.rs
@@ -65,6 +65,11 @@
 
 extern crate ucd_trie;
 
+#[cfg(feature = "pretty-print")]
+extern crate serde;
+#[cfg(feature = "pretty-print")]
+extern crate serde_json;
+
 pub use parser::Parser;
 pub use parser_state::{state, Atomicity, Lookahead, MatchDir, ParseResult, ParserState};
 pub use position::Position;


Test patch: