Repository: raphlinus/pulldown-cmark (rust)

Issue:
[fuzz result] setext header in middle of link definition
This code:

```markdown
[first
-
second]: https://example.com

[first
-
second]
```

Should parse to this text:

> [first
> -
> second]: https://example.com
>
> [first
> -
> second]

Instead, it parses the top as a link def.

<details>

Events from pulldown-cmark:

```rust
"\n[*\n-\n*]:^\n" -> [
]
```

Events from pandoc:

```rust
"\n[*\n-\n*]:^\n" -> [
  Start(Heading { level: H2, id: None, classes: [], attrs: [] })
    Text(Boxed("[*"))
  End(Heading(H2))
  Start(Paragraph)
    Text(Boxed("*]:^"))
  End(Paragraph)
]
```

Events from commonmark.js:

```rust
"\n[*\n-\n*]:^\n" -> [
  Start(Heading { level: H2, id: None, classes: [], attrs: [] })
    Text(Boxed("[*"))
  End(Heading(H2))
  Start(Paragraph)
    Text(Boxed("*]:^"))
  End(Paragraph)
]
```

</details>


Pull request description:
Allow setext headers to interrupt link def insides
Fixes #789

This is the same parse result from [commonmark.js], [commonmark-hs], [markdown-it], and [GitHub].

[commonmark.js]: https://spec.commonmark.org/dingus/?text=%5Bfirst%0A-%0Asecond%5D%3A%20https%3A%2F%2Fexample.com%0A%0A%5Bfirst%0A-%0Asecond%5D
[commonmark-hs]: https://pandoc.org/try/?params=%7B%22text%22%3A%22%5Bfirst%5Cn-%5Cnsecond%5D%3A+https%3A%2F%2Fexample.com%5Cn%5Cn%5Bfirst%5Cn-%5Cnsecond%5D%22%2C%22to%22%3A%22html5%22%2C%22from%22%3A%22commonmark%22%2C%22standalone%22%3Afalse%2C%22embed-resources%22%3Afalse%2C%22table-of-contents%22%3Afalse%2C%22number-sections%22%3Afalse%2C%22citeproc%22%3Afalse%2C%22html-math-method%22%3A%22plain%22%2C%22wrap%22%3A%22auto%22%2C%22highlight-style%22%3Anull%2C%22files%22%3A%7B%7D%2C%22template%22%3Anull%7D
[markdown-it]: https://markdown-it.github.io/#md3=%7B%22source%22%3A%22%5Bfirst%5Cn-%5Cnsecond%5D%3A%20https%3A%2F%2Fexample.com%5Cn%5Cn%5Bfirst%5Cn-%5Cnsecond%5D%22%2C%22defaults%22%3A%7B%22html%22%3Afalse%2C%22xhtmlOut%22%3Afalse%2C%22breaks%22%3Afalse%2C%22langPrefix%22%3A%22language-%22%2C%22linkify%22%3Atrue%2C%22typographer%22%3Atrue%2C%22_highlight%22%3Atrue%2C%22_strict%22%3Afalse%2C%22_view%22%3A%22html%22%7D%7D
[GitHub]: https://gist.github.com/notriddle/312e9c855d0c822d56b25fe604c04eae

Patch:
diff --git a/specs/regression.txt b/specs/regression.txt
index 12e5c03..c66fa0f 100644
--- a/specs/regression.txt
+++ b/specs/regression.txt
@@ -1870,3 +1870,118 @@ List can interrupt the paragraph at the start of a link definition if it starts
 <p>- b</p>
 <p>List can interrupt the paragraph at the start of a link definition if it starts with three spaces, but not four.</p>
 ````````````````````````````````
+
+ISSUE 789
+
+```````````````````````````````` example
+[first
+-
+second]: https://example.com
+
+[first
+-
+second]
+.
+<h2>[first</h2>
+<p>second]: https://example.com</p>
+<h2>[first</h2>
+<p>second]</p>
+````````````````````````````````
+
+```````````````````````````````` example
+[first
+    -
+second]: https://example.com
+
+[first
+    -
+second]
+.
+<p><a href="https://example.com">first
+-
+second</a></p>
+````````````````````````````````
+
+```````````````````````````````` example
+[first]: https://example.com
+"
+-
+"
+
+[first]
+.
+<h2>"</h2>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+````````````````````````````````
+
+```````````````````````````````` example
+[first]: https://example.com
+"
+    -
+"
+
+[first]
+.
+<p><a href="https://example.com" title="
+-
+">first</a></p>
+````````````````````````````````
+
+```````````````````````````````` example
+> [first]: https://example.com
+> "
+> -
+> "
+>
+> [first]
+.
+<blockquote>
+<h2>"</h2>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+</blockquote>
+````````````````````````````````
+
+```````````````````````````````` example
+> [first]: https://example.com
+> "
+>     -
+> "
+>
+> [first]
+.
+<blockquote>
+<p><a href="https://example.com" title="
+-
+">first</a></p>
+</blockquote>
+````````````````````````````````
+
+```````````````````````````````` example
+[first]: https://example.com
+"
+\
+"
+
+[first]
+.
+<p><a href="https://example.com" title="
+\
+">first</a></p>
+````````````````````````````````
+
+```````````````````````````````` example
+[first]: https://example.com
+"
+\
+
+"
+
+[first]
+.
+<p>"
+\</p>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+````````````````````````````````
diff --git a/src/firstpass.rs b/src/firstpass.rs
index b65a96f..dbd2e0d 100644
--- a/src/firstpass.rs
+++ b/src/firstpass.rs
@@ -1393,9 +1393,12 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                 &mut line_start,
                 self.options.has_gfm_footnotes(),
             ) == self.tree.spine_len();
+            if line_start.scan_space(4) {
+                return Some(line_start.bytes_scanned());
+            }
             let bytes_scanned = line_start.bytes_scanned();
             let suffix = &bytes[bytes_scanned..];
-            if self.scan_paragraph_interrupt(suffix, current_container) {
+            if self.scan_paragraph_interrupt(suffix, current_container) || (current_container && scan_setext_heading(suffix).is_some()) {
                 None
             } else {
                 Some(bytes_scanned)
@@ -1451,6 +1454,83 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         Some((i, newlines))
     }
 
+    // returns (bytelength, title_str)
+    fn scan_refdef_title<'t>(&self, text: &'t str) -> Option<(usize, CowStr<'t>)> {
+        let bytes = text.as_bytes();
+        let closing_delim = match bytes.get(0)? {
+            b'\'' => b'\'',
+            b'"' => b'"',
+            b'(' => b')',
+            _ => return None,
+        };
+        let mut bytecount = 1;
+        let mut linestart = 1;
+    
+        let mut linebuf = None;
+    
+        while let Some(&c) = bytes.get(bytecount) {
+            match c {
+                b'\n' | b'\r' => {
+                    // push text to line buffer
+                    // this is used to strip the block formatting:
+                    //
+                    // > [first]: http://example.com "
+                    // > second"
+                    //
+                    // should get turned into `<a href="example.com" title=" second">first</a>`
+                    let linebuf = if let Some(linebuf) = &mut linebuf {
+                        linebuf
+                    } else {
+                        linebuf = Some(String::new());
+                        linebuf.as_mut().unwrap()
+                    };
+                    linebuf.push_str(&text[linestart..bytecount]);
+                    linebuf.push('\n'); // normalize line breaks
+                    // skip line break
+                    bytecount += 1;
+                    if c == b'\r' && bytes.get(bytecount) == Some(&b'\n') {
+                        bytecount += 1;
+                    }
+                    let mut line_start = LineStart::new(&bytes[bytecount..]);
+                    let current_container = scan_containers(
+                        &self.tree,
+                        &mut line_start,
+                        self.options.has_gfm_footnotes(),
+                    ) == self.tree.spine_len();
+                    if !line_start.scan_space(4) {
+                        let suffix = &bytes[bytecount + line_start.bytes_scanned()..];
+                        if self.scan_paragraph_interrupt(suffix, current_container) || scan_setext_heading(suffix).is_some() {
+                            return None;
+                        }
+                    }
+                    bytecount += line_start.bytes_scanned();
+                    linestart = bytecount;
+                    if scan_blank_line(&bytes[bytecount..]).is_some() {
+                        // blank line - not allowed
+                        return None;
+                    }
+
+                }
+                b'\\' => {
+                    bytecount += 1;
+                    if let Some(c) = bytes.get(bytecount) {
+                        if c != &b'\r' && c != &b'\n' {
+                            bytecount += 1;
+                        }
+                    }
+                }
+                c if c == closing_delim => {
+                    let cow = linebuf.map(CowStr::from).unwrap_or(CowStr::from(&text[linestart..bytecount]));
+                    return Some((bytecount + 1, cow));
+                }
+                _ => {
+                    bytecount += 1;
+                }
+            }
+        }
+        None
+    }
+
     /// Returns # of bytes and definition.
     /// Assumes the label of the reference including colon has already been scanned.
     fn scan_refdef(&self, span_start: usize, start: usize) -> Option<(usize, LinkDef<'a>)> {
@@ -1496,7 +1576,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
 
         // scan title
         // if this fails but newline == 1, return also a refdef without title
-        if let Some((title_length, title)) = scan_refdef_title(&self.text[i..]) {
+        if let Some((title_length, title)) = self.scan_refdef_title(&self.text[i..]) {
             i += title_length;
             backup.1.span = span_start..i;
             backup.1.title = Some(unescape(title));
diff --git a/src/scanners.rs b/src/scanners.rs
index 34b543e..9052d8e 100644
--- a/src/scanners.rs
+++ b/src/scanners.rs
@@ -828,47 +828,6 @@ pub(crate) fn scan_entity(bytes: &[u8]) -> (usize, Option<CowStr<'static>>) {
     (0, None)
 }
 
-// FIXME: we can most likely re-use other scanners
-// returns (bytelength, title_str)
-pub(crate) fn scan_refdef_title(text: &str) -> Option<(usize, &str)> {
-    let mut chars = text.chars().peekable();
-    let closing_delim = match chars.next()? {
-        '\'' => '\'',
-        '"' => '"',
-        '(' => ')',
-        _ => return None,
-    };
-    let mut bytecount = 1;
-
-    while let Some(c) = chars.next() {
-        match c {
-            '\n' => {
-                bytecount += 1;
-                let mut next = *chars.peek()?;
-                while is_ascii_whitespace_no_nl(next as u8) {
-                    bytecount += chars.next()?.len_utf8();
-                    next = *chars.peek()?;
-                }
-                if *chars.peek()? == '\n' {
-                    // blank line - not allowed
-                    return None;
-                }
-            }
-            '\\' => {
-                let next_char = chars.next()?;
-                bytecount += 1 + next_char.len_utf8();
-            }
-            c if c == closing_delim => {
-                return Some((bytecount + 1, &text[1..bytecount]));
-            }
-            c => {
-                bytecount += c.len_utf8();
-            }
-        }
-    }
-    None
-}
-
 // note: dest returned is raw, still needs to be unescaped
 // TODO: check that nested parens are really not allowed for refdefs
 // TODO(performance): this func should probably its own unescaping
@@ -1044,7 +1003,8 @@ fn scan_attribute_value(
 }
 
 // Remove backslash escapes and resolve entities
-pub(crate) fn unescape(input: &str) -> CowStr<'_> {
+pub(crate) fn unescape<'a, I: Into<CowStr<'a>>>(input: I) -> CowStr<'a> {
+    let input = input.into();
     let mut result = String::new();
     let mut mark = 0;
     let mut i = 0;


Test patch:
diff --git a/tests/suite/regression.rs b/tests/suite/regression.rs
index 1cb8068..ec3dbd1 100644
--- a/tests/suite/regression.rs
+++ b/tests/suite/regression.rs
@@ -2073,16 +2073,22 @@ foo)
 <p><a href="foo">link</a></p>
 </blockquote>
 "##;
-  
+
     test_markdown_html(original, expected, false, false, false);
 }
-  
+
 #[test]
 fn regression_test_131() {
-    let original = r##"[linkme]: foo
-    - baz
+    let original = r##"[link](foo
+)
+
+> [link](foo
+> )
 "##;
-    let expected = r##"<p>- baz</p>
+    let expected = r##"<p><a href="foo">link</a></p>
+<blockquote>
+<p><a href="foo">link</a></p>
+</blockquote>
 "##;
 
     test_markdown_html(original, expected, false, false, false);
@@ -2091,22 +2097,56 @@ fn regression_test_131() {
 #[test]
 fn regression_test_132() {
     let original = r##"[link](foo
+"bar"
 )
 
 > [link](foo
+> "bar"
 > )
 "##;
-    let expected = r##"<p><a href="foo">link</a></p>
+    let expected = r##"<p><a href="foo" title="bar">link</a></p>
 <blockquote>
-<p><a href="foo">link</a></p>
+<p><a href="foo" title="bar">link</a></p>
 </blockquote>
 "##;
- 
-  test_markdown_html(original, expected, false, false, false);
+
+    test_markdown_html(original, expected, false, false, false);
 }
-  
-  #[test]
+
+#[test]
 fn regression_test_133() {
+    let original = r##"[link](	
+	foo	
+	"bar"	
+	)
+
+> [link](	
+> 	foo	
+> 	"bar"	
+> 	)
+"##;
+    let expected = r##"<p><a href="foo" title="bar">link</a></p>
+<blockquote>
+<p><a href="foo" title="bar">link</a></p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_134() {
+    let original = r##"[linkme]: foo
+    - baz
+"##;
+    let expected = r##"<p>- baz</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_135() {
     let original = r##"[linkme-3]:
    [^foo]:
 
@@ -2124,26 +2164,7 @@ GFM footnotes can interrupt link defs if they have three spaces, but not four.
 }
 
 #[test]
-fn regression_test_134() {
-    let original = r##"[link](foo
-"bar"
-)
-
-> [link](foo
-> "bar"
-> )
-"##;
-    let expected = r##"<p><a href="foo" title="bar">link</a></p>
-<blockquote>
-<p><a href="foo" title="bar">link</a></p>
-</blockquote>
-"##;
-
-    test_markdown_html(original, expected, false, false, false);
-}
-
-  #[test]
-fn regression_test_135() {
+fn regression_test_136() {
     let original = r##"[linkme-3]:
    ===
 
@@ -2160,7 +2181,7 @@ Setext heading can interrupt link def if it has three spaces, but not four.
 }
 
 #[test]
-fn regression_test_136() {
+fn regression_test_137() {
     let original = r##"[linkme-3]: a
    - a
 
@@ -2178,3 +2199,148 @@ List can interrupt the paragraph at the start of a link definition if it starts
 
     test_markdown_html(original, expected, false, false, false);
 }
+
+#[test]
+fn regression_test_138() {
+    let original = r##"[first
+-
+second]: https://example.com
+
+[first
+-
+second]
+"##;
+    let expected = r##"<h2>[first</h2>
+<p>second]: https://example.com</p>
+<h2>[first</h2>
+<p>second]</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_139() {
+    let original = r##"[first
+    -
+second]: https://example.com
+
+[first
+    -
+second]
+"##;
+    let expected = r##"<p><a href="https://example.com">first
+-
+second</a></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_140() {
+    let original = r##"[first]: https://example.com
+"
+-
+"
+
+[first]
+"##;
+    let expected = r##"<h2>"</h2>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_141() {
+    let original = r##"[first]: https://example.com
+"
+    -
+"
+
+[first]
+"##;
+    let expected = r##"<p><a href="https://example.com" title="
+-
+">first</a></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_142() {
+    let original = r##"> [first]: https://example.com
+> "
+> -
+> "
+>
+> [first]
+"##;
+    let expected = r##"<blockquote>
+<h2>"</h2>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_143() {
+    let original = r##"> [first]: https://example.com
+> "
+>     -
+> "
+>
+> [first]
+"##;
+    let expected = r##"<blockquote>
+<p><a href="https://example.com" title="
+-
+">first</a></p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_144() {
+    let original = r##"[first]: https://example.com
+"
+\
+"
+
+[first]
+"##;
+    let expected = r##"<p><a href="https://example.com" title="
+\
+">first</a></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_145() {
+    let original = r##"[first]: https://example.com
+"
+\
+
+"
+
+[first]
+"##;
+    let expected = r##"<p>"
+\</p>
+<p>"</p>
+<p><a href="https://example.com">first</a></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}