Repository: pulldown-cmark/pulldown-cmark (rust)

Issue:
`>` counts as both the start of a blockquote and the end of inline HTML?
Using commit: 2f183d71d12c810754ade2138488f3e2551f40b8.

I'm a little confused about this case, and I think this is an issue with identifying inline HTML. Given the following input:

```markdown
* <c
  >

```

pulldown-cmark considers the `\n>` as part of the `InlineHtml` text and as starting a blockquote.

running `cargo run -- -e` produces these events:

```
0..10: Start(List(None))
0..10: Start(Item)
2..8: InlineHtml(Boxed("<c\n>"))
8..9: Start(BlockQuote(None))
8..9: End(BlockQuote(None))
0..10: End(Item)
0..10: End(List(false))
EOF
```

Pull request description:
Use an explicit node for tight paragraphs
This cleans up a lot of hacks, because our tree didn't quite match the way commonmark actually works, and we need to avoid letting a block appear as a child of an inline.

It also fixes a bug in HTML parsing that could only be hit with tight lists.

Fixes #1002

Patch:
diff --git a/pulldown-cmark/specs/regression.txt b/pulldown-cmark/specs/regression.txt
index 8d55c80..5d2dcda 100644
--- a/pulldown-cmark/specs/regression.txt
+++ b/pulldown-cmark/specs/regression.txt
@@ -2788,3 +2788,13 @@ the trailing space after the &gt; should be stripped
 .
 <p><a href="Wiki%3C">Link</a></p>
 ````````````````````````````````
+
+```````````````````````````````` example_wikilinks
+* <c
+  >
+.
+<ul>
+<li>&lt;c
+<blockquote></blockquote></li>
+</ul>
+````````````````````````````````
diff --git a/pulldown-cmark/src/firstpass.rs b/pulldown-cmark/src/firstpass.rs
index 9c1fc22..5fa1258 100644
--- a/pulldown-cmark/src/firstpass.rs
+++ b/pulldown-cmark/src/firstpass.rs
@@ -174,6 +174,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                         item,
                         Item {
                             body: ItemBody::Paragraph
+                                | ItemBody::TightParagraph
                                 | ItemBody::MaybeDefinitionListTitle
                                 | ItemBody::DefinitionListDefinition(_),
                             ..
@@ -190,7 +191,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                 })
             {
                 match item.body {
-                    ItemBody::Paragraph => {
+                    ItemBody::Paragraph | ItemBody::TightParagraph => {
                         item.body = ItemBody::DefinitionList(true);
                         let Item { start, end, .. } = *item;
                         let list_idx = self.tree.cur().unwrap();
@@ -2188,44 +2189,15 @@ fn get_html_end_tag(text_bytes: &[u8]) -> Option<&'static str> {
     }
 }
 
-// https://english.stackexchange.com/a/285573
 fn surgerize_tight_list(tree: &mut Tree<Item>, list_ix: TreeIndex) {
     let mut list_item = tree[list_ix].child;
     while let Some(listitem_ix) = list_item {
-        // first child is special, controls how we repoint list_item.child
-        let list_item_firstborn = tree[listitem_ix].child;
-
-        // Check that list item has children - this is not necessarily the case!
-        if let Some(firstborn_ix) = list_item_firstborn {
-            if let ItemBody::Paragraph = tree[firstborn_ix].item.body {
-                tree[listitem_ix].child = tree[firstborn_ix].child;
-            }
-
-            let mut list_item_child = Some(firstborn_ix);
-            let mut node_to_repoint = None;
-            while let Some(child_ix) = list_item_child {
-                // surgerize paragraphs
-                let repoint_ix = if let ItemBody::Paragraph = tree[child_ix].item.body {
-                    if let Some(child_firstborn) = tree[child_ix].child {
-                        if let Some(repoint_ix) = node_to_repoint {
-                            tree[repoint_ix].next = Some(child_firstborn);
-                        }
-                        let mut child_lastborn = child_firstborn;
-                        while let Some(lastborn_next_ix) = tree[child_lastborn].next {
-                            child_lastborn = lastborn_next_ix;
-                        }
-                        child_lastborn
-                    } else {
-                        child_ix
-                    }
-                } else {
-                    child_ix
-                };
-
-                node_to_repoint = Some(repoint_ix);
-                tree[repoint_ix].next = tree[child_ix].next;
-                list_item_child = tree[child_ix].next;
+        let mut node_ix = tree[listitem_ix].child;
+        while let Some(node) = node_ix {
+            if let ItemBody::Paragraph = tree[node].item.body {
+                tree[node].item.body = ItemBody::TightParagraph;
             }
+            node_ix = tree[node].next;
         }
 
         list_item = tree[listitem_ix].next;
diff --git a/pulldown-cmark/src/parse.rs b/pulldown-cmark/src/parse.rs
index cec1bf5..e3d9f2a 100644
--- a/pulldown-cmark/src/parse.rs
+++ b/pulldown-cmark/src/parse.rs
@@ -101,6 +101,7 @@ pub(crate) enum ItemBody {
 
     // These are block items.
     Paragraph,
+    TightParagraph,
     Rule,
     Heading(HeadingLevel, Option<HeadingIndex>), // heading level
     FencedCodeBlock(CowIndex),
@@ -173,9 +174,6 @@ impl ItemBody {
                 | HardBreak(..)
         )
     }
-    fn is_block(&self) -> bool {
-        !self.is_inline()
-    }
 }
 
 #[derive(Debug)]
@@ -525,12 +523,6 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
                                     can_close,
                                 );
                             }
-                            if self.tree[scan_ix].item.body.is_block() {
-                                // If this is a tight list, blocks and inlines might be
-                                // siblings. Inlines can't cross the boundary like that.
-                                scan = None;
-                                break;
-                            }
                             scan = self.tree[scan_ix].next;
                         }
                         scan
@@ -587,12 +579,6 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
                                     self.code_delims.insert(delim_count, scan_ix);
                                 }
                             }
-                            if self.tree[scan_ix].item.body.is_block() {
-                                // If this is a tight list, blocks and inlines might be
-                                // siblings. Inlines can't cross the boundary like that.
-                                scan = None;
-                                break;
-                            }
                             scan = self.tree[scan_ix].next;
                         }
                         if scan.is_none() {
@@ -864,15 +850,7 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
                         }
                     }
                 }
-                _ => {
-                    // If this is a tight list, blocks and inlines might be
-                    // siblings. Inlines can't cross the boundary like that.
-                    if let Some(cur_ix) = cur {
-                        if self.tree[cur_ix].item.body.is_block() {
-                            self.link_stack.clear();
-                        }
-                    }
-                }
+                _ => {}
             }
             prev = cur;
             cur = self.tree[cur_ix].next;
@@ -1132,13 +1110,6 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
                 }
                 _ => {
                     prev = cur;
-                    // If this is a tight list, blocks and inlines might be
-                    // siblings. Inlines can't cross the boundary like that.
-                    if let Some(cur_ix) = cur {
-                        if self.tree[cur_ix].item.body.is_block() {
-                            self.inline_stack.pop_all(&mut self.tree);
-                        }
-                    }
                     cur = self.tree[cur_ix].next;
                 }
             }
@@ -2207,6 +2178,13 @@ impl<'a, F: BrokenLinkCallback<'a>> Iterator for OffsetIter<'a, F> {
         match self.inner.tree.cur() {
             None => {
                 let ix = self.inner.tree.pop()?;
+                let ix = if matches!(self.inner.tree[ix].item.body, ItemBody::TightParagraph) {
+                    // tight paragraphs emit nothing
+                    self.inner.tree.next_sibling(ix);
+                    return self.next();
+                } else {
+                    ix
+                };
                 let tag_end = body_to_tag_end(&self.inner.tree[ix].item.body);
                 self.inner.tree.next_sibling(ix);
                 let span = self.inner.tree[ix].item.start..self.inner.tree[ix].item.end;
@@ -2214,6 +2192,14 @@ impl<'a, F: BrokenLinkCallback<'a>> Iterator for OffsetIter<'a, F> {
                 Some((Event::End(tag_end), span))
             }
             Some(cur_ix) => {
+                let cur_ix =
+                    if matches!(self.inner.tree[cur_ix].item.body, ItemBody::TightParagraph) {
+                        // tight paragraphs emit nothing
+                        self.inner.tree.push();
+                        self.inner.tree.cur().unwrap()
+                    } else {
+                        cur_ix
+                    };
                 if self.inner.tree[cur_ix].item.body.is_maybe_inline() {
                     self.inner.handle_inline();
                 }
@@ -2363,11 +2349,25 @@ impl<'a, F: BrokenLinkCallback<'a>> Iterator for Parser<'a, F> {
         match self.tree.cur() {
             None => {
                 let ix = self.tree.pop()?;
+                let ix = if matches!(self.tree[ix].item.body, ItemBody::TightParagraph) {
+                    // tight paragraphs emit nothing
+                    self.tree.next_sibling(ix);
+                    return self.next();
+                } else {
+                    ix
+                };
                 let tag_end = body_to_tag_end(&self.tree[ix].item.body);
                 self.tree.next_sibling(ix);
                 Some(Event::End(tag_end))
             }
             Some(cur_ix) => {
+                let cur_ix = if matches!(self.tree[cur_ix].item.body, ItemBody::TightParagraph) {
+                    // tight paragraphs emit nothing
+                    self.tree.push();
+                    self.tree.cur().unwrap()
+                } else {
+                    cur_ix
+                };
                 if self.tree[cur_ix].item.body.is_maybe_inline() {
                     self.handle_inline();
                 }


Test patch:
diff --git a/pulldown-cmark/tests/suite/regression.rs b/pulldown-cmark/tests/suite/regression.rs
index 3839734..54fe14a 100644
--- a/pulldown-cmark/tests/suite/regression.rs
+++ b/pulldown-cmark/tests/suite/regression.rs
@@ -3325,3 +3325,17 @@ fn regression_test_208() {
 
     test_markdown_html(original, expected, false, false, false, false, true);
 }
+
+#[test]
+fn regression_test_209() {
+    let original = r##"* <c
+  >
+"##;
+    let expected = r##"<ul>
+<li>&lt;c
+<blockquote></blockquote></li>
+</ul>
+"##;
+
+    test_markdown_html(original, expected, false, false, false, false, true);
+}