Repository: pulldown-cmark/pulldown-cmark (rust)

Issue:
[regression in 0.10] Consecutive footnote refs fail to parse correctly
Consider the following markdown:

```
Lorem ipsum.[^a][^b]

[^a]: Foo
[^b]: Bar
```

Which on GitHub, renders thus:

Lorem ipsum.[^a][^b]

[^a]: Foo
[^b]: Bar

With pulldown-cmark **v0.9.6**, these were correctly parsed as two consecutive footnote references:

```
➜  cargo run -- --events --enable-footnotes < in.md
   Compiling pulldown-cmark v0.9.6 (/home/zoni/Workspace/pulldown-cmark)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.61s
     Running `/home/zoni/Workspace/pulldown-cmark/target/debug/pulldown-cmark --events --enable-footnotes`
0..21: Start(Paragraph)
0..12: Text(Borrowed("Lorem ipsum."))
12..16: FootnoteReference(Borrowed("a"))         // <--- footnote ref a
16..20: FootnoteReference(Borrowed("b"))         // <--- footnote ref b
0..21: End(Paragraph)
22..32: Start(FootnoteDefinition(Borrowed("a")))
28..31: Start(Paragraph)
28..31: Text(Borrowed("Foo"))
31..32: SoftBreak
28..31: End(Paragraph)
22..32: End(FootnoteDefinition(Borrowed("a")))
32..42: Start(FootnoteDefinition(Borrowed("b")))
38..42: Start(Paragraph)
38..41: Text(Borrowed("Bar"))
38..42: End(Paragraph)
32..42: End(FootnoteDefinition(Borrowed("b")))
EOF
```

With pulldown-cmark **0.11** (`v0.11.0-44-g4e9bc2c`),  this fails to resolve both footnote references correctly. Notably, it fails on both the **old**:

```
➜  cargo run -- --events --enable-old-footnotes < in.md
   Compiling pulldown-cmark v0.11.0 (/home/zoni/Workspace/pulldown-cmark/pulldown-cmark)
   Compiling pulldown-cmark-escape v0.11.0 (/home/zoni/Workspace/pulldown-cmark/pulldown-cmark-escape)
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.63s
     Running `/home/zoni/Workspace/pulldown-cmark/target/debug/pulldown-cmark --events --enable-old-footnotes`
0..21: Start(Paragraph)
0..12: Text(Borrowed("Lorem ipsum."))
12..13: Text(Borrowed("["))                    // <---- footnote ref a incorrectly parsed as text
13..15: Text(Borrowed("^a"))
15..16: Text(Borrowed("]"))
16..20: FootnoteReference(Borrowed("b"))       // <---- footnote ref b
0..21: End(Paragraph)
22..42: Start(FootnoteDefinition(Borrowed("a")))
28..42: Start(Paragraph)
28..31: Text(Borrowed("Foo"))
31..32: SoftBreak
32..36: FootnoteReference(Borrowed("b"))
36..41: Text(Borrowed(": Bar"))
28..42: End(Paragraph)
22..42: End(FootnoteDefinition)
EOF
```

As well as the **new** (GFM) footnote parser:

```
➜  cargo run -- --events --enable-footnotes < in.md
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.07s
     Running `/home/zoni/Workspace/pulldown-cmark/target/debug/pulldown-cmark --events --enable-footnotes`
0..21: Start(Paragraph)
0..12: Text(Borrowed("Lorem ipsum."))
12..13: Text(Borrowed("["))                    // <---- footnote ref a incorrectly parsed as text
13..15: Text(Borrowed("^a"))
15..16: Text(Borrowed("]"))
16..20: FootnoteReference(Borrowed("b"))       // <---- footnote ref b
0..21: End(Paragraph)
22..32: Start(FootnoteDefinition(Borrowed("a")))
28..32: Start(Paragraph)
28..31: Text(Borrowed("Foo"))
28..32: End(Paragraph)
22..32: End(FootnoteDefinition)
32..42: Start(FootnoteDefinition(Borrowed("b")))
38..42: Start(Paragraph)
38..41: Text(Borrowed("Bar"))
38..42: End(Paragraph)
32..42: End(FootnoteDefinition)
EOF
```

(Putting a space between `[^a]` and `[^b]` will produce correct events. The problem exists when two or more footnotes are squished up together. Then, only the last one is parsed correctly)

Pull request:
Update old footnote format to interrupt paragraph
Keeps parity with v0.9.6.

Fixes #927

Patch:
diff --git a/pulldown-cmark/specs/old_footnotes.txt b/pulldown-cmark/specs/old_footnotes.txt
index 32f7ab4..22aded4 100644
--- a/pulldown-cmark/specs/old_footnotes.txt
+++ b/pulldown-cmark/specs/old_footnotes.txt
@@ -145,8 +145,9 @@ They do need one line between each other.
 [^Doh]: I know. Wrong Doe. And it will render right.
 [^1]: Common for people practicing music.
 .
-<p><sup class="footnote-reference"><a href="#Doh">1</a></sup> Ray Me Fa So La Te Do! <sup class="footnote-reference"><a href="#1">2</a></sup></p><div class="footnote-definition" id="Doh"><sup class="footnote-definition-label">1</sup><p>I know. Wrong Doe. And it will render right.
-<sup class="footnote-reference"><a href="#1">2</a></sup>: Common for people practicing music.</p></div>
+<p><sup class="footnote-reference"><a href="#Doh">1</a></sup> Ray Me Fa So La Te Do! <sup class="footnote-reference"><a href="#1">2</a></sup></p>
+<div class="footnote-definition" id="Doh"><sup class="footnote-definition-label">1</sup><p>I know. Wrong Doe. And it will render right.</p></div>
+<div class="footnote-definition" id="1"><sup class="footnote-definition-label">2</sup><p>Common for people practicing music.</p></div>
 ````````````````````````````````
 
 Second check for footnotes:
diff --git a/pulldown-cmark/src/firstpass.rs b/pulldown-cmark/src/firstpass.rs
index 59d3eb5..97c8bdb 100644
--- a/pulldown-cmark/src/firstpass.rs
+++ b/pulldown-cmark/src/firstpass.rs
@@ -490,7 +490,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         if scan_paragraph_interrupt_no_table(
             &bytes[ix..],
             current_container,
-            self.options.has_gfm_footnotes(),
+            self.options.contains(Options::ENABLE_FOOTNOTES),
             &self.tree,
         ) {
             return None;
@@ -1607,13 +1607,13 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         }
         i += 1;
         self.finish_list(start);
-        if self.options.has_gfm_footnotes() {
-            if let Some(node_ix) = self.tree.peek_up() {
-                if let ItemBody::FootnoteDefinition(..) = self.tree[node_ix].item.body {
-                    // finish previous footnote if it's still open
-                    self.pop(start);
-                }
+        if let Some(node_ix) = self.tree.peek_up() {
+            if let ItemBody::FootnoteDefinition(..) = self.tree[node_ix].item.body {
+                // finish previous footnote if it's still open
+                self.pop(start);
             }
+        }
+        if self.options.has_gfm_footnotes() {
             i += scan_whitespace_no_nl(&bytes[i..]);
         }
         self.allocs.footdefs.0.insert(
@@ -1862,8 +1862,8 @@ impl<'a, 'b> FirstPass<'a, 'b> {
 
     /// Checks whether we should break a paragraph on the given input.
     fn scan_paragraph_interrupt(&self, bytes: &[u8], current_container: bool) -> bool {
-        let gfm_footnote = self.options.has_gfm_footnotes();
-        if scan_paragraph_interrupt_no_table(bytes, current_container, gfm_footnote, &self.tree) {
+        let has_footnote = self.options.contains(Options::ENABLE_FOOTNOTES);
+        if scan_paragraph_interrupt_no_table(bytes, current_container, has_footnote, &self.tree) {
             return true;
         }
         // pulldown-cmark allows heavy tables, that have a `|` on the header row,
@@ -2018,7 +2018,7 @@ fn count_header_cols(
 fn scan_paragraph_interrupt_no_table(
     bytes: &[u8],
     current_container: bool,
-    gfm_footnote: bool,
+    has_footnote: bool,
     tree: &Tree<Item>,
 ) -> bool {
     scan_eol(bytes).is_some()
@@ -2036,7 +2036,7 @@ fn scan_paragraph_interrupt_no_table(
         })
         || bytes.starts_with(b"<")
             && (get_html_end_tag(&bytes[1..]).is_some() || starts_html_block_type_6(&bytes[1..]))
-        || (gfm_footnote
+        || (has_footnote
             && bytes.starts_with(b"[^")
             && scan_link_label_rest(
                 std::str::from_utf8(&bytes[2..]).unwrap(),


Test patch:
diff --git a/pulldown-cmark/tests/suite/old_footnotes.rs b/pulldown-cmark/tests/suite/old_footnotes.rs
index 706f682..9924f9f 100644
--- a/pulldown-cmark/tests/suite/old_footnotes.rs
+++ b/pulldown-cmark/tests/suite/old_footnotes.rs
@@ -154,8 +154,9 @@ fn old_footnotes_test_8() {
 [^Doh]: I know. Wrong Doe. And it will render right.
 [^1]: Common for people practicing music.
 "##;
-    let expected = r##"<p><sup class="footnote-reference"><a href="#Doh">1</a></sup> Ray Me Fa So La Te Do! <sup class="footnote-reference"><a href="#1">2</a></sup></p><div class="footnote-definition" id="Doh"><sup class="footnote-definition-label">1</sup><p>I know. Wrong Doe. And it will render right.
-<sup class="footnote-reference"><a href="#1">2</a></sup>: Common for people practicing music.</p></div>
+    let expected = r##"<p><sup class="footnote-reference"><a href="#Doh">1</a></sup> Ray Me Fa So La Te Do! <sup class="footnote-reference"><a href="#1">2</a></sup></p>
+<div class="footnote-definition" id="Doh"><sup class="footnote-definition-label">1</sup><p>I know. Wrong Doe. And it will render right.</p></div>
+<div class="footnote-definition" id="1"><sup class="footnote-definition-label">2</sup><p>Common for people practicing music.</p></div>
 "##;
 
     test_markdown_html(original, expected, false, false, true);