Repository: kivikakk/comrak (rust)

Issue:
Description List as an extension
There is an open [thread about description lists](https://talk.commonmark.org/t/description-list/289) in [talk.commonmark.org](https://talk.commonmark.org/). Since comrak has support for [some extensions](https://github.com/kivikakk/comrak/blob/dfe55ae29482c038c0ba2f4adf925421c6709008/src/parser/mod.rs#L136-L254), I think that it would be nice to add support for definition lists.

## Syntax

There is no official syntax for definition list, but it is implemented in many Markdown processors. One of the comments includes [a list of lins](https://talk.commonmark.org/t/description-list/289/11) to the documentation of some of those processors.

The most common syntax is using colon at the beginning of the definition.

    term1

    : definition1

    term2

    : definition2


This syntax is used in [Pandoc](https://pandoc.org/MANUAL.html#definition-lists), [PHP Markdown](https://michelf.ca/projects/php-markdown/extra/#def-list), [fletcher/MultiMarkdown-4](https://fletcher.github.io/MultiMarkdown-4/definitionlists), [Python Markdown](https://python-markdown.github.io/extensions/definition_lists/), [kramdown](https://kramdown.gettalong.org/quickref.html#definition-lists), etc.



Discussion:
Awesome! I'd happily accept a PR that implemented this. 😊 

Patch:
diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -46,9 +46,9 @@ FLAGS:
 
 OPTIONS:
         --default-info-string <INFO>    Default value for fenced code block's info strings if none is given
-    -e, --extension <EXTENSION>...      Specify an extension name to use [values: strikethrough, tagfilter, table,
-                                        autolink, tasklist, superscript, footnotes]
-    -t, --to <FORMAT>                   Specify output format [default: html]  [values: html, commonmark]
+    -e, --extension <EXTENSION>...      Specify an extension name to use [possible values: strikethrough, tagfilter,
+                                        table, autolink, tasklist, superscript, footnotes, description-lists]
+    -t, --to <FORMAT>                   Specify output format [default: html]  [possible values: html, commonmark]
         --header-ids <PREFIX>           Use the Comrak header IDs extension, with the given ID prefix
         --width <WIDTH>                 Specify wrap width (0 = nowrap) [default: 0]
 
@@ -127,7 +127,7 @@ Comrak supports the five extensions to CommonMark defined in the
 * [Autolinks](https://github.github.com/gfm/#autolinks-extension-)
 * [Disallowed Raw HTML](https://github.github.com/gfm/#disallowed-raw-html-extension-)
 
-as well as superscript and footnotes.
+as well as superscript, footnotes, and description lists.
 
 By default none are enabled; they are individually enabled with each parse by
 setting the appropriate values in the
diff --git a/examples/s-expr.rs b/examples/s-expr.rs
new file mode 100644
--- /dev/null
+++ b/examples/s-expr.rs
@@ -0,0 +1,115 @@
+// s-expr
+//
+// Parse CommonMark source files and print their AST as S-expressions.
+//
+// # Usage
+//
+//  $ cargo run --example s-expr file1.md file2.md ...
+//  $ cat file.md | cargo run --example s-expr
+
+extern crate comrak;
+
+/// Spaces to indent nested nodes
+const INDENT: usize = 4;
+
+/// If true, the close parenthesis is printed in its own line.
+const CLOSE_NEWLINE: bool = false;
+
+use comrak::{parse_document, Arena, ComrakOptions};
+use comrak::nodes::{AstNode, NodeValue};
+use std::env;
+use std::error::Error;
+use std::fs::File;
+use std::io::{self, BufWriter, Read, Write};
+
+fn iter_nodes<'a, W: Write>(
+    node: &'a AstNode<'a>,
+    writer: &mut W,
+    indent: usize,
+) -> io::Result<()> {
+
+    use NodeValue::*;
+
+    macro_rules! try_node_inline {
+        ($node:expr, $name:ident) => ({
+            if let $name(t) = $node {
+                return write!(writer, concat!(stringify!($name), "({:?})"), String::from_utf8_lossy(&t));
+            }
+        })
+    }
+
+    match &node.data.borrow().value {
+        Text(t) => write!(writer, "{:?}", String::from_utf8_lossy(&t))?,
+        value => {
+            try_node_inline!(value, Code);
+            try_node_inline!(value, FootnoteDefinition);
+            try_node_inline!(value, FootnoteReference);
+            try_node_inline!(value, HtmlInline);
+
+            let has_blocks = node.children().any(|c| c.data.borrow().value.block());
+
+            write!(writer, "({:?}", value)?;
+            for child in node.children() {
+                if has_blocks {
+                    write!(writer, "\n{1:0$}", indent + INDENT, " ")?;
+                } else {
+                    write!(writer, " ")?;
+                }
+                iter_nodes(child, writer, indent + INDENT)?;
+            }
+
+            if indent == 0 {
+                write!(writer, "\n)\n")?;
+            } else if CLOSE_NEWLINE && has_blocks {
+                write!(writer, "\n{1:0$})", indent, " ")?;
+            } else {
+                write!(writer, ")")?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn dump(source: &str) -> io::Result<()> {
+
+    let arena = Arena::new();
+
+    let opts = ComrakOptions {
+        ext_strikethrough: true,
+        ext_tagfilter: true,
+        ext_table: true,
+        ext_autolink: true,
+        ext_tasklist: true,
+        ext_superscript: true,
+        ext_footnotes: true,
+        ext_description_lists: true,
+        ..ComrakOptions::default()
+    };
+
+    let doc = parse_document(&arena, source, &opts);
+
+    let mut output = BufWriter::new(io::stdout());
+    iter_nodes(doc, &mut output, 0)
+}
+
+fn main() -> Result<(), Box<Error>> {
+
+    let mut args = env::args_os().skip(1).peekable();
+    let mut body = String::new();
+
+    if args.peek().is_none() {
+        io::stdin().read_to_string(&mut body)?;
+        dump(&body)?;
+    }
+
+    for filename in args {
+        println!("{:?}", filename);
+
+        body.clear();
+        File::open(&filename)?.read_to_string(&mut body)?;
+        dump(&body)?;
+    }
+
+    Ok(())
+}
diff --git a/src/cm.rs b/src/cm.rs
--- a/src/cm.rs
+++ b/src/cm.rs
@@ -335,6 +335,10 @@ impl<'a, 'o> CommonMarkFormatter<'a, 'o> {
                     self.cr();
                 }
             }
+            NodeValue::DescriptionList => (),
+            NodeValue::DescriptionItem(..) => (),
+            NodeValue::DescriptionTerm => (),
+            NodeValue::DescriptionDetails => if entering { write!(self, ": ").unwrap() },
             NodeValue::Heading(ref nch) => if entering {
                 for _ in 0..nch.level {
                     write!(self, "#").unwrap();
diff --git a/src/html.rs b/src/html.rs
--- a/src/html.rs
+++ b/src/html.rs
@@ -348,6 +348,23 @@ impl<'o> HtmlFormatter<'o> {
             } else {
                 try!(self.output.write_all(b"</li>\n"));
             },
+            NodeValue::DescriptionList => if entering {
+                try!(self.cr());
+                try!(self.output.write_all(b"<dl>"));
+            } else {
+                try!(self.output.write_all(b"</dl>\n"));
+            },
+            NodeValue::DescriptionItem(..) => (),
+            NodeValue::DescriptionTerm => if entering {
+                try!(self.output.write_all(b"<dt>"));
+            } else {
+                try!(self.output.write_all(b"</dt>\n"));
+            },
+            NodeValue::DescriptionDetails => if entering {
+                try!(self.output.write_all(b"<dd>"));
+            } else {
+                try!(self.output.write_all(b"</dd>\n"));
+            },
             NodeValue::Heading(ref nch) => {
                 lazy_static! {
                     static ref REJECTED_CHARS: Regex = Regex::new(r"[^\p{L}\p{M}\p{N}\p{Pc} -]").unwrap();
diff --git a/src/main.rs b/src/main.rs
--- a/src/main.rs
+++ b/src/main.rs
@@ -64,6 +64,7 @@ fn main() {
                     "tasklist",
                     "superscript",
                     "footnotes",
+                    "description-lists",
                 ])
                 .value_name("EXTENSION")
                 .help("Specify an extension name to use"),
@@ -125,6 +126,7 @@ fn main() {
         ext_superscript: exts.remove("superscript"),
         ext_header_ids: matches.value_of("header-ids").map(|s| s.to_string()),
         ext_footnotes: matches.is_present("footnotes"),
+        ext_description_lists: exts.remove("description-lists"),
     };
 
     assert!(exts.is_empty());
diff --git a/src/nodes.rs b/src/nodes.rs
--- a/src/nodes.rs
+++ b/src/nodes.rs
@@ -33,6 +33,31 @@ pub enum NodeValue {
     /// **blocks**.
     Item(NodeList),
 
+    /// **Block**. A description list, enabled with `ext_description_lists` option.  Contains
+    /// description items.
+    ///
+    /// It is required to put a blank line between terms and details.
+    ///
+    /// ``` md
+    /// Term 1
+    ///
+    /// : Details 1
+    ///
+    /// Term 2
+    ///
+    /// : Details 2
+    /// ```
+    DescriptionList,
+
+    /// *Block**. An item of a description list.  Contains a term and one details block.
+    DescriptionItem(NodeDescriptionItem),
+
+    /// **Block**. Term of an item in a definition list.
+    DescriptionTerm,
+
+    /// **Block**. Details of an item in a definition list.
+    DescriptionDetails,
+
     /// **Block**. A code block; may be [fenced](https://github.github.com/gfm/#fenced-code-blocks)
     /// or [indented](https://github.github.com/gfm/#indented-code-blocks).  Contains raw text
     /// which is not parsed as Markdown, although is HTML escaped.
@@ -167,6 +192,16 @@ pub struct NodeList {
     pub tight: bool,
 }
 
+/// The metadata of a description list
+#[derive(Debug, Default, Clone, Copy)]
+pub struct NodeDescriptionItem {
+    #[doc(hidden)]
+    pub marker_offset: usize,
+
+    #[doc(hidden)]
+    pub padding: usize,
+}
+
 /// The type of list.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum ListType {
@@ -253,6 +288,10 @@ impl NodeValue {
             | NodeValue::BlockQuote
             | NodeValue::FootnoteDefinition(_)
             | NodeValue::List(..)
+            | NodeValue::DescriptionList
+            | NodeValue::DescriptionItem(_)
+            | NodeValue::DescriptionTerm
+            | NodeValue::DescriptionDetails
             | NodeValue::Item(..)
             | NodeValue::CodeBlock(..)
             | NodeValue::HtmlBlock(..)
@@ -356,6 +395,8 @@ pub fn can_contain_type<'a>(node: &'a AstNode<'a>, child: &NodeValue) -> bool {
         NodeValue::Document
         | NodeValue::BlockQuote
         | NodeValue::FootnoteDefinition(_)
+        | NodeValue::DescriptionTerm
+        | NodeValue::DescriptionDetails
         | NodeValue::Item(..) => {
             child.block() && match *child {
                 NodeValue::Item(..) => false,
@@ -368,6 +409,16 @@ pub fn can_contain_type<'a>(node: &'a AstNode<'a>, child: &NodeValue) -> bool {
             _ => false,
         },
 
+        NodeValue::DescriptionList => match *child {
+            NodeValue::DescriptionItem(_) => true,
+            _ => false,
+        },
+
+        NodeValue::DescriptionItem(_) => match *child {
+            NodeValue::DescriptionTerm | NodeValue::DescriptionDetails => true,
+            _ => false,
+        },
+
         NodeValue::Paragraph
         | NodeValue::Heading(..)
         | NodeValue::Emph
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -6,8 +6,8 @@ use arena_tree::Node;
 use ctype::{isdigit, isspace};
 use entity;
 use nodes;
-use nodes::{make_block, Ast, AstNode, ListDelimType, ListType, NodeCodeBlock, NodeHeading,
-            NodeHtmlBlock, NodeList, NodeValue};
+use nodes::{make_block, Ast, AstNode, ListDelimType, ListType, NodeCodeBlock, NodeDescriptionItem,
+            NodeHeading, NodeHtmlBlock, NodeList, NodeValue};
 use regex::bytes::Regex;
 use scanners;
 use std::cell::RefCell;
@@ -21,6 +21,15 @@ use typed_arena::Arena;
 const TAB_STOP: usize = 4;
 const CODE_INDENT: usize = 4;
 
+macro_rules! node_matches {
+    ($node:expr, $pat:pat) => ({
+        match $node.data.borrow().value {
+            $pat => true,
+            _ => false,
+        }
+    })
+}
+
 /// Parse a Markdown document to an AST.
 ///
 /// See the documentation of the crate root for an example.
@@ -286,6 +295,34 @@ pub struct ComrakOptions {
     ///            "<p>Hi<sup class=\"footnote-ref\"><a href=\"#fn1\" id=\"fnref1\">[1]</a></sup>.</p>\n<section class=\"footnotes\">\n<ol>\n<li id=\"fn1\">\n<p>A greeting. <a href=\"#fnref1\" class=\"footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n");
     /// ```
     pub ext_footnotes: bool,
+
+    /// Enables the description lists extension.
+    ///
+    /// Each term must be defined in one paragraph, followed by a blank line,
+    /// and then by the details.  Details begins with a colon.
+    ///
+    /// ``` md
+    /// First term
+    ///
+    /// : Details for the **first term**
+    ///
+    /// Second term
+    ///
+    /// : Details for the **second term**
+    ///
+    ///     More details in second paragraph.
+    /// ```
+    ///
+    /// ```
+    /// # use comrak::{markdown_to_html, ComrakOptions};
+    /// let options = ComrakOptions {
+    ///   ext_description_lists: true,
+    ///   ..ComrakOptions::default()
+    /// };
+    /// assert_eq!(markdown_to_html("Term\n\n: Definition", &options),
+    ///            "<dl><dt>\n<p>Term</p>\n</dt>\n<dd>\n<p>Definition</p>\n</dd>\n</dl>\n");
+    /// ```
+    pub ext_description_lists: bool,
 }
 
 #[derive(Clone)]
@@ -488,6 +525,9 @@ impl<'a, 'o> Parser<'a, 'o> {
                 NodeValue::Item(ref nl) => if !self.parse_node_item_prefix(line, container, nl) {
                     return (false, container, should_continue);
                 },
+                NodeValue::DescriptionItem(ref di) => if !self.parse_description_item_prefix(line, container, di) {
+                    return (false, container, should_continue);
+                },
                 NodeValue::CodeBlock(..) => {
                     if !self.parse_code_block_prefix(line, container, ast, &mut should_continue) {
                         return (false, container, should_continue);
@@ -645,6 +685,15 @@ impl<'a, 'o> Parser<'a, 'o> {
                     *container,
                     NodeValue::FootnoteDefinition(c.to_vec()),
                 );
+            } else if !indented && self.options.ext_description_lists
+                && line[self.first_nonspace] == b':'
+                && self.parse_desc_list_details(container)
+            {
+                let offset = self.first_nonspace + 1 - self.offset;
+                self.advance_offset(line, offset, false);
+                if strings::is_space_or_tab(line[self.offset]) {
+                    self.advance_offset(line, 1, true);
+                }
             } else if (!indented || match container.data.borrow().value {
                 NodeValue::List(..) => true,
                 _ => false,
@@ -803,6 +852,24 @@ impl<'a, 'o> Parser<'a, 'o> {
         }
     }
 
+    fn parse_description_item_prefix(
+        &mut self,
+        line: &[u8],
+        container: &'a AstNode<'a>,
+        di: &NodeDescriptionItem,
+    ) -> bool {
+        if self.indent >= di.marker_offset + di.padding {
+            self.advance_offset(line, di.marker_offset + di.padding, true);
+            true
+        } else if self.blank && container.first_child().is_some() {
+            let offset = self.first_nonspace - self.offset;
+            self.advance_offset(line, offset, false);
+            true
+        } else {
+            false
+        }
+    }
+
     fn parse_code_block_prefix(
         &mut self,
         line: &[u8],
@@ -864,6 +931,51 @@ impl<'a, 'o> Parser<'a, 'o> {
         }
     }
 
+    fn parse_desc_list_details(&mut self, container: &mut &'a AstNode<'a>) -> bool {
+        let last_child = match container.last_child() {
+            Some(lc) => lc,
+            None => return false,
+        };
+
+        if node_matches!(last_child, NodeValue::Paragraph) {
+            // We have found the details after the paragraph for the term.
+            //
+            // This paragraph is moved as a child of a new DescriptionTerm node.
+            //
+            // If the node before the paragraph is a description list, the item
+            // is added to it. If not, create a new list.
+
+            last_child.detach();
+
+            let list = match container.last_child() {
+                Some(lc) if node_matches!(lc, NodeValue::DescriptionList) => {
+                    reopen_ast_nodes(lc);
+                    lc
+                }
+                _ => {
+                    self.add_child(container, NodeValue::DescriptionList)
+                }
+            };
+
+            let metadata = NodeDescriptionItem {
+                marker_offset: self.indent,
+                padding: 2,
+            };
+
+            let item = self.add_child(list, NodeValue::DescriptionItem(metadata));
+            let term = self.add_child(item, NodeValue::DescriptionTerm);
+            let details = self.add_child(item, NodeValue::DescriptionDetails);
+
+            term.append(last_child);
+
+            *container = details;
+
+            true
+        } else {
+            false
+        }
+    }
+
     fn add_child(
         &mut self,
         mut parent: &'a AstNode<'a>,
@@ -1549,6 +1661,16 @@ fn lists_match(list_data: &NodeList, item_data: &NodeList) -> bool {
         && list_data.bullet_char == item_data.bullet_char
 }
 
+fn reopen_ast_nodes<'a>(mut ast: &'a AstNode<'a>) {
+    loop {
+        ast.data.borrow_mut().open = true;
+        ast = match ast.parent() {
+            Some(p) => p,
+            None => return,
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum AutolinkType {
     URI,


Test patch:
diff --git a/src/tests.rs b/src/tests.rs
--- a/src/tests.rs
+++ b/src/tests.rs
@@ -898,3 +898,68 @@ fn nul_replacement_4() {
 fn nul_replacement_5() {
     html("a\r\n\0b", "<p>a\n\u{fffd}b</p>\n");
 }
+
+#[test]
+fn description_lists() {
+    html_opts(
+        concat!(
+            "Term 1\n",
+            "\n",
+            ": Definition 1\n",
+            "\n",
+            "Term 2 with *inline markup*\n",
+            "\n",
+            ": Definition 2\n"
+        ),
+        concat!(
+            "<dl>",
+            "<dt>\n",
+            "<p>Term 1</p>\n",
+            "</dt>\n",
+            "<dd>\n",
+            "<p>Definition 1</p>\n",
+            "</dd>\n",
+            "<dt>\n",
+            "<p>Term 2 with <em>inline markup</em></p>\n",
+            "</dt>\n",
+            "<dd>\n",
+            "<p>Definition 2</p>\n",
+            "</dd>\n",
+            "</dl>\n",
+        ),
+        |opts| opts.ext_description_lists = true,
+    );
+
+    html_opts(
+        concat!(
+            "* Nested\n",
+            "\n",
+            "    Term 1\n\n",
+            "    :   Definition 1\n\n",
+            "    Term 2 with *inline markup*\n\n",
+            "    :   Definition 2\n\n"
+        ),
+        concat!(
+            "<ul>\n",
+            "<li>\n",
+            "<p>Nested</p>\n",
+            "<dl>",
+            "<dt>\n",
+            "<p>Term 1</p>\n",
+            "</dt>\n",
+            "<dd>\n",
+            "<p>Definition 1</p>\n",
+            "</dd>\n",
+            "<dt>\n",
+            "<p>Term 2 with <em>inline markup</em></p>\n",
+            "</dt>\n",
+            "<dd>\n",
+            "<p>Definition 2</p>\n",
+            "</dd>\n",
+            "</dl>\n",
+            "</li>\n",
+            "</ul>\n",
+        ),
+        |opts| opts.ext_description_lists = true,
+    );
+}