Repository: yshavit/mdq
Pull request #35 (2024-06-16 21:06:06)

add `inlines_to_plain_string`
Also, get rid of `mod fmt_json`. The actual to-json is something I should think about harder and do right, or not at all. The to-plaintext helper is now in fmt_str.

I had originally added the json formatting as a quick way of getting some output, before I had good markdown rendering. I don't need that shim anymore.


Patch:
diff --git a/src/fmt_json.rs b/src/fmt_json.rs
deleted file mode 100644
index 1858b97..0000000
--- a/src/fmt_json.rs
+++ /dev/null
@@ -1,180 +0,0 @@
-use std::borrow::Borrow;
-
-use markdown::mdast::AlignKind;
-use serde_json::{json, Map, Value};
-
-use crate::tree::*;
-
-const BODY_KEY: &str = "body";
-
-pub fn nodes_to_json<N, R>(nodes: &[N]) -> Value
-where
-    N: Borrow<MdqNode>,
-    R: InlineResolver,
-{
-    Value::Array(nodes.iter().map(|n| node_to_json::<R>(n.borrow())).collect())
-}
-
-pub fn node_to_json<R: InlineResolver>(node: &MdqNode) -> Value {
-    match node {
-        MdqNode::Root(Root { body }) => to_jsons::<R>(body),
-        MdqNode::Header(Header {
-            depth,
-            title,
-            body: contents,
-        }) => json!({
-            "header": json!({
-                    "title": R::inlines_to_value(title),
-                    "depth": json!(depth),
-                    BODY_KEY: to_jsons::<R>(contents),
-            })
-        }),
-        MdqNode::Paragraph(Paragraph { body }) => json!({"paragraph": R::inlines_to_value(body)}),
-        MdqNode::BlockQuote(BlockQuote { body }) => json!({"block_quote": to_jsons::<R>(body)}),
-        MdqNode::List(List { starting_index, items }) => {
-            let mut as_map = Map::new();
-            match starting_index {
-                Some(start_idx) => {
-                    as_map.insert("type".to_string(), json!("ordered"));
-                    as_map.insert("start_at".to_string(), json!(start_idx));
-                    true
-                }
-                None => {
-                    as_map.insert("type".to_string(), json!("unordered"));
-                    false
-                }
-            };
-            let items = items
-                .iter()
-                .map(|li| {
-                    json!({
-                        "checked": li.checked.clone(),
-                        "item": to_jsons::<R>(&li.item),
-                    })
-                })
-                .collect();
-            as_map.insert("items".to_string(), Value::Array(items));
-
-            json!({"list": Value::Object(as_map)})
-        }
-        MdqNode::Table(Table {
-            alignments: align,
-            rows,
-        }) => {
-            let aligns: Vec<Option<&str>> = align
-                .iter()
-                .map(|a| match a {
-                    AlignKind::Left => Some("left"),
-                    AlignKind::Right => Some("right"),
-                    AlignKind::Center => Some("center"),
-                    AlignKind::None => None,
-                })
-                .collect();
-            json!({
-                "table": {
-                    "column_alignments": json!(aligns),
-                    "rows": rows.iter().map(|row| {
-                        row.iter().map(|col|
-                            R::inlines_to_value(col)
-                        ).collect::<Vec<_>>()
-                    }).collect::<Vec<_>>()
-                }
-            })
-        }
-        MdqNode::ThematicBreak => {
-            json!({"thematic_break": Value::Null})
-        }
-        MdqNode::CodeBlock(CodeBlock { variant, value }) => match variant {
-            CodeVariant::Code(opts) => {
-                let (lang, meta) = match opts {
-                    None => (None, None),
-                    Some(CodeOpts { language, metadata }) => (Some(language.to_string()), metadata.to_owned()),
-                };
-                json!({
-                    "code":json!({
-                        "language": lang,
-                        "metadata": meta,
-                        BODY_KEY: json!(value),
-                    })
-                })
-            }
-            CodeVariant::Math { metadata } => json!({
-                "math": json!({
-                    "metadata": metadata,
-                    BODY_KEY: json!(value),
-                })
-            }),
-            CodeVariant::Toml => json!({"toml": json!(value)}),
-            CodeVariant::Yaml => json!({"yaml": json!(value)}),
-        },
-        MdqNode::Inline(inline) => R::inlines_to_value(&[inline]),
-    }
-}
-
-pub struct TextOnly {
-    _private: bool,
-}
-
-impl TextOnly {
-    pub fn line_to_string<I>(line: &[I]) -> String
-    where
-        I: Borrow<Inline>,
-    {
-        let mut str = String::new();
-        Self::build_strings(&mut str, line);
-        str
-    }
-
-    fn build_strings<I>(out: &mut String, line: &[I])
-    where
-        I: Borrow<Inline>,
-    {
-        for i in line {
-            Self::build_string(out, i.borrow())
-        }
-    }
-
-    fn build_string(out: &mut String, elem: &Inline) {
-        match elem {
-            Inline::Span { children, .. } => {
-                for child in children {
-                    Self::build_string(out, child);
-                }
-            }
-            Inline::Text { value, .. } => out.push_str(&value),
-            Inline::Link { text, .. } => {
-                for child in text {
-                    Self::build_string(out, child);
-                }
-            }
-            Inline::Image {
-                link: Link { title, .. },
-                ..
-            } => {
-                out.push_str("<image");
-                if let Some(title) = title {
-                    out.push_str(": ");
-                    out.push_str(title);
-                }
-                out.push('>');
-            }
-            Inline::Footnote { .. } => {
-                // ignore
-            }
-        }
-    }
-}
-
-impl InlineResolver for TextOnly {
-    fn inlines_to_value<I: Borrow<Inline>>(inlines: &[I]) -> Value {
-        Value::String(Self::line_to_string(inlines))
-    }
-}
-
-pub trait InlineResolver {
-    fn inlines_to_value<I: Borrow<Inline>>(inlines: &[I]) -> Value;
-}
-
-fn to_jsons<R: InlineResolver>(nodes: &[MdqNode]) -> Value {
-    Value::Array(nodes.iter().map(|n| node_to_json::<R>(n)).collect())
-}
diff --git a/src/fmt_str.rs b/src/fmt_str.rs
new file mode 100644
index 0000000..86668d1
--- /dev/null
+++ b/src/fmt_str.rs
@@ -0,0 +1,142 @@
+use crate::tree::{Inline, InlineVariant};
+use std::borrow::Borrow;
+
+pub fn inlines_to_plain_string<N: Borrow<Inline>>(inlines: &[N]) -> String {
+    let mut result = String::with_capacity(inlines.len() * 10); // random guess
+    build_inlines(&mut result, inlines);
+
+    result
+}
+
+fn build_inlines<N: Borrow<Inline>>(out: &mut String, inlines: &[N]) {
+    for inline in inlines {
+        build_inline(out, inline.borrow());
+    }
+}
+
+fn build_inline(out: &mut String, elem: &Inline) {
+    match elem {
+        Inline::Span { children, .. } => build_inlines(out, children),
+        Inline::Text { variant, value, .. } => {
+            if !matches!(variant, InlineVariant::Html) {
+                out.push_str(value)
+            }
+        }
+        Inline::Link { text, .. } => build_inlines(out, text),
+        Inline::Image { alt, .. } => out.push_str(alt),
+        Inline::Footnote(footnote) => {
+            out.push_str("[^");
+            out.push_str(&footnote.label);
+            out.push(']');
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use indoc::indoc;
+
+    use crate::tree::{Inline, InlineVariant, MdqNode, SpanVariant};
+    use crate::unwrap;
+    use crate::utils_for_test::VariantsChecker;
+    use lazy_static::lazy_static;
+    use markdown::ParseOptions;
+
+    lazy_static! {
+        static ref VARIANTS_CHECKER: VariantsChecker<Inline> = crate::new_variants_checker!(Inline {
+            Span { variant: SpanVariant::Delete, .. },
+            Span { variant: SpanVariant::Emphasis, .. },
+            Span { variant: SpanVariant::Strong, .. },
+            Text { variant: InlineVariant::Text, .. },
+            Text { variant: InlineVariant::Code, .. },
+            Text { variant: InlineVariant::Math, .. },
+            Text { variant: InlineVariant::Html, .. },
+            Link { .. },
+            Image { .. },
+            Footnote(_),
+        });
+    }
+
+    #[test]
+    fn spans() {
+        check("_hello world_", "hello world");
+        check("**hello world**", "hello world");
+        check("~~hello world~~", "hello world");
+    }
+
+    #[test]
+    fn texts() {
+        check("hello world", "hello world");
+        check("`hello world`", "hello world");
+        check("$hello world$", "hello world");
+        // html is covered separately, since it isn't wrapped in a paragraph: see issue #34
+    }
+
+    #[test]
+    fn text_html() {
+        let node = markdown::to_mdast("<foo>", &ParseOptions::gfm()).unwrap();
+        let mdq_node: MdqNode = node.try_into().unwrap();
+        unwrap!(mdq_node, MdqNode::Root(root));
+        unwrap!(&root.body[0], MdqNode::Inline(inline));
+        VARIANTS_CHECKER.see(inline);
+        let actual = inlines_to_plain_string(&[inline]);
+        assert_eq!(&actual, "");
+    }
+
+    #[test]
+    fn links() {
+        check("[foo](https://example.com)", "foo");
+        check("[foo _with emphasis_](https://example.com)", "foo with emphasis");
+        check(
+            indoc! {r#"
+            [foo][1]
+
+            [1]: https://example.com"#},
+            "foo",
+        )
+    }
+
+    #[test]
+    fn images() {
+        check("![foo](https://example.com)", "foo");
+        check("![foo _with emphasis_](https://example.com)", "foo with emphasis"); // md is ignored in alt
+        check(
+            indoc! {r#"
+            ![foo][1]
+
+            [1]: https://example.com"#},
+            "foo",
+        )
+    }
+
+    #[test]
+    fn footnote() {
+        check(
+            indoc! {r#"
+            [^1]
+
+            [^1]: my footnote"#},
+            "[^1]",
+        )
+    }
+
+    #[test]
+    fn all_variants_checked() {
+        VARIANTS_CHECKER.wait_for_all();
+    }
+
+    /// Because this is such simple functionality, we're just going to do a simple end-to-end test from original
+    /// markdown to plain text.
+    fn check(md: &str, expect: &str) {
+        let mut options = ParseOptions::gfm();
+        options.constructs.math_text = true;
+        let node = markdown::to_mdast(md, &options).unwrap();
+        let mdq_node: MdqNode = node.try_into().unwrap();
+        unwrap!(mdq_node, MdqNode::Root(root));
+        unwrap!(&root.body[0], MdqNode::Paragraph(p));
+        p.body.iter().for_each(|inline| VARIANTS_CHECKER.see(inline));
+        let actual = inlines_to_plain_string(&p.body);
+        assert_eq!(&actual, expect);
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index b6d7c4a..90cedcd 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,13 +2,12 @@ use std::io;
 use std::io::{stdin, Read};
 use std::string::ToString;
 
-use crate::fmt_json::TextOnly;
 use crate::fmt_md::MdOptions;
 use crate::output::Stream;
 use crate::tree::MdqNode;
 
-mod fmt_json;
 mod fmt_md;
+mod fmt_str;
 mod output;
 mod select;
 mod str_utils;
@@ -32,9 +31,6 @@ fn main() {
 
     let found = selector.find(&mdq);
 
-    let jsons = fmt_json::nodes_to_json::<_, TextOnly>(&found);
-    println!("{}", jsons);
-    out.write_str("\n\n=======================================\n\n");
     fmt_md::write_md(&MdOptions::default(), &mut out, &found);
     out.write_str("\n");
 }
diff --git a/src/select.rs b/src/select.rs
index a54734d..ee62c6b 100644
--- a/src/select.rs
+++ b/src/select.rs
@@ -1,4 +1,4 @@
-use crate::fmt_json;
+use crate::fmt_str::inlines_to_plain_string;
 use crate::tree::*;
 
 #[allow(dead_code)]
@@ -69,7 +69,7 @@ impl Selector {
             MdqNode::Root(Root { body }) => SelectResult::Recurse(body),
             MdqNode::Header(Header { title, body, .. }) => {
                 if let Selector::Heading(matcher) = self {
-                    if matcher.matches(&Self::line_to_string(title)) {
+                    if matcher.matches(&inlines_to_plain_string(title)) {
                         SelectResult::Found(body.iter().map(|elem| elem).collect())
                     } else {
                         SelectResult::Recurse(body)
@@ -145,8 +145,4 @@ impl Selector {
     fn find_in_children<'a>(&'a self, children: &'a Vec<MdqNode>) -> Vec<&MdqNode> {
         children.iter().flat_map(|elem| self.find(elem)).collect()
     }
-
-    fn line_to_string(line: &[Inline]) -> String {
-        fmt_json::TextOnly::line_to_string(line)
-    }
 }
diff --git a/src/tree.rs b/src/tree.rs
index e381f43..92b26ee 100644
--- a/src/tree.rs
+++ b/src/tree.rs
@@ -185,9 +185,10 @@ pub enum SpanVariant {
     Strong,
 }
 
+// TODO rename to TextVariant
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub enum InlineVariant {
-    Text,
+    Text, // TODO rename to Plain
     Code,
     Math,
     Html,
@@ -702,6 +703,7 @@ mod tests {
     ///
     /// For example, footnote are `[^a]` in markdown; does that identifier get parsed as `"^a"` or `"a"`?
     mod all_nodes {
+        use crate::unwrap;
         use crate::utils_for_test::VariantsChecker;
         use indoc::indoc;
         use lazy_static::lazy_static;
@@ -710,18 +712,6 @@ mod tests {
 
         use super::*;
 
-        /// Turn a pattern match into an `if let ... { else panic! }`.
-        macro_rules! unwrap {
-            // TODO intellij errors if these are in the other order. file a ticket.
-            ($enum_value:expr, $enum_variant:pat) => {
-                let node = $enum_value;
-                let node_debug = format!("{:?}", node);
-                let $enum_variant = node else {
-                    panic!("Expected {} but saw {}", stringify!($enum_variant), node_debug);
-                };
-            };
-        }
-
         macro_rules! check {
             (no_node: $enum_value:expr, $enum_variant:pat, $lookups:expr => $no_node:expr $(, $body:block)? ) => {{
                 let node = $enum_value;


Test patch:
diff --git a/src/utils_for_test.rs b/src/utils_for_test.rs
index 8f64c3e..8baf3ec 100644
--- a/src/utils_for_test.rs
+++ b/src/utils_for_test.rs
@@ -7,6 +7,19 @@ pub use test_utils::*;
 mod test_utils {
     use std::{thread, time};
 
+    /// Turn a pattern match into an `if let ... { else panic! }`.
+    #[macro_export]
+    macro_rules! unwrap {
+        // TODO intellij errors if these are in the other order. file a ticket.
+        ($enum_value:expr, $enum_variant:pat) => {
+            let node = $enum_value;
+            let node_debug = format!("{:?}", node);
+            let $enum_variant = node else {
+                panic!("Expected {} but saw {}", stringify!($enum_variant), node_debug);
+            };
+        };
+    }
+
     pub struct VariantsChecker<E> {
         require: std::sync::Arc<std::sync::Mutex<std::collections::HashSet<String>>>,
         resolver: fn(&E) -> &str,