Repository: raphlinus/pulldown-cmark (rust)

Issue:
Parsing `"\\\n\n"` loses the backslash
Compare parsing `"\\\n"` and `"\\\n\n"`:

```rust
"\\\n" -> [
  Start(Paragraph)
    Text(Borrowed("\\"))
  End(Paragraph)
]
"\\\n\n" -> [
  Start(Paragraph)
  End(Paragraph)
]
```

The backslash disappears when it is followed by two newlines.

The commonmark.js tool includes the backslash in both cases. The output of `echo -n '\\\n\n' | commonmark --to xml` looks like this:

```xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE document SYSTEM "CommonMark.dtd">
<document xmlns="http://commonmark.org/xml/1.0">
  <paragraph>
    <text>\</text>
  </paragraph>
</document>
```


Pull request:
Include trailing backslash at all break points in paragraph
Fixes #658


Patch:
diff --git a/specs/regression.txt b/specs/regression.txt
index e006d1a..f906385 100644
--- a/specs/regression.txt
+++ b/specs/regression.txt
@@ -1124,3 +1124,59 @@ ISSUE 540
 </li>
 </ul>
 ````````````````````````````````
+
+
+ISSUE 658
+
+```````````````````````````````` example
+a\
+
+b
+.
+<p>a\</p>
+<p>b</p>
+````````````````````````````````
+
+```````````````````````````````` example
+a\
+* b
+.
+<p>a\</p>
+<ul>
+<li>b</li>
+</ul>
+````````````````````````````````
+
+```````````````````````````````` example
+a\
+> b
+.
+<p>a\</p>
+<blockquote>
+<p>b</p>
+</blockquote>
+````````````````````````````````
+
+```````````````````````````````` example
+a\
+# b
+.
+<p>a\</p>
+<h1>b</h1>
+````````````````````````````````
+
+```````````````````````````````` example
+a\
+==
+.
+<h1>a\</h1>
+````````````````````````````````
+
+```````````````````````````````` example
+> a\
+>
+.
+<blockquote>
+<p>a\</p>
+</blockquote>
+````````````````````````````````
diff --git a/specs/table.txt b/specs/table.txt
index e68137d..7bc9b3d 100644
--- a/specs/table.txt
+++ b/specs/table.txt
@@ -422,14 +422,30 @@ consistent with GFM and Pandoc.
 | pandoc                | paragraph |
 | multimarkdown         | table     |
 
-
 ```````````````````````````````` example
 a | b\
 - | -
 1 | 2
 .
-<p>a | b</p>
+<p>a | b\</p>
 <ul>
 <li>| - 1 | 2</li>
 </ul>
 ````````````````````````````````
+
+
+As a block structure, table parsing has a higher priority than backslashes.
+This is consistent with how other paragraph-interrupting structures work,
+and is what GitHub does.
+
+```````````````````````````````` example
+a\
+| b | c |
+|---|---|
+| d | e |
+.
+<p>a\</p>
+<table><thead><tr><th>b</th><th>c</th></tr></thead><tbody>
+<tr><td>d</td><td>e</td></tr>
+</tbody></table>
+````````````````````````````````
diff --git a/src/firstpass.rs b/src/firstpass.rs
index 2b6e9d3..c6e0bb0 100644
--- a/src/firstpass.rs
+++ b/src/firstpass.rs
@@ -375,17 +375,17 @@ impl<'a, 'b> FirstPass<'a, 'b> {
             let mut line_start = LineStart::new(&bytes[ix..]);
             let current_container =
                 scan_containers(&self.tree, &mut line_start, self.options.has_gfm_footnotes()) == self.tree.spine_len();
+            let trailing_backslash_pos = match brk {
+                Some(Item {
+                    start,
+                    body: ItemBody::HardBreak,
+                    ..
+                }) if bytes[start] == b'\\' => Some(start),
+                _ => None,
+            };
             if !line_start.scan_space(4) {
                 let ix_new = ix + line_start.bytes_scanned();
                 if current_container {
-                    let trailing_backslash_pos = match brk {
-                        Some(Item {
-                            start,
-                            body: ItemBody::HardBreak,
-                            ..
-                        }) if bytes[start] == b'\\' => Some(start),
-                        _ => None,
-                    };
                     if let Some(ix_setext) =
                         self.parse_setext_heading(ix_new, node_ix, trailing_backslash_pos.is_some())
                     {
@@ -399,11 +399,17 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                 // first check for non-empty lists, then for other interrupts
                 let suffix = &bytes[ix_new..];
                 if scan_paragraph_interrupt(suffix, current_container, self.options.has_gfm_footnotes()) {
+                    if let Some(pos) = trailing_backslash_pos {
+                        self.tree.append_text(pos, pos + 1);
+                    }
                     break;
                 }
             }
             line_start.scan_all_space();
             if line_start.is_at_eol() {
+                if let Some(pos) = trailing_backslash_pos {
+                    self.tree.append_text(pos, pos + 1);
+                }
                 break;
             }
             ix = next_ix + line_start.bytes_scanned();
@@ -496,21 +502,6 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                     let mut i = ix;
                     let eol_bytes = scan_eol(&bytes[ix..]).unwrap();
 
-                    let end_ix = ix + eol_bytes;
-                    let trailing_backslashes = scan_rev_while(&bytes[..ix], |b| b == b'\\');
-                    if trailing_backslashes % 2 == 1 && end_ix < bytes_len {
-                        i -= 1;
-                        self.tree.append_text(begin_text, i);
-                        return LoopInstruction::BreakAtWith(
-                            end_ix,
-                            Some(Item {
-                                start: i,
-                                end: end_ix,
-                                body: ItemBody::HardBreak,
-                            }),
-                        );
-                    }
-
                     // If tables are not enabled yet and the next char is a pipe,
                     // and there was content before then a table begins here but
                     // there was content before the table:
@@ -527,6 +518,21 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                         return LoopInstruction::BreakAtWith(ix, None);
                     }
 
+                    let end_ix = ix + eol_bytes;
+                    let trailing_backslashes = scan_rev_while(&bytes[..ix], |b| b == b'\\');
+                    if trailing_backslashes % 2 == 1 && end_ix < bytes_len {
+                        i -= 1;
+                        self.tree.append_text(begin_text, i);
+                        return LoopInstruction::BreakAtWith(
+                            end_ix,
+                            Some(Item {
+                                start: i,
+                                end: end_ix,
+                                body: ItemBody::HardBreak,
+                            }),
+                        );
+                    }
+
                     if mode == TableParseMode::Scan && pipes > 0 {
                         // check if we may be parsing a table
                         let next_line_ix = ix + eol_bytes;


Test patch:
diff --git a/tests/suite/regression.rs b/tests/suite/regression.rs
index 3bbda8a..071cc03 100644
--- a/tests/suite/regression.rs
+++ b/tests/suite/regression.rs
@@ -1313,3 +1313,80 @@ fn regression_test_85() {
 
     test_markdown_html(original, expected, false, false, false);
 }
+
+#[test]
+fn regression_test_86() {
+    let original = r##"a\
+
+b
+"##;
+    let expected = r##"<p>a\</p>
+<p>b</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_87() {
+    let original = r##"a\
+* b
+"##;
+    let expected = r##"<p>a\</p>
+<ul>
+<li>b</li>
+</ul>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_88() {
+    let original = r##"a\
+> b
+"##;
+    let expected = r##"<p>a\</p>
+<blockquote>
+<p>b</p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_89() {
+    let original = r##"a\
+# b
+"##;
+    let expected = r##"<p>a\</p>
+<h1>b</h1>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_90() {
+    let original = r##"a\
+==
+"##;
+    let expected = r##"<h1>a\</h1>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_91() {
+    let original = r##"> a\
+>
+"##;
+    let expected = r##"<blockquote>
+<p>a\</p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
diff --git a/tests/suite/table.rs b/tests/suite/table.rs
index 1aa58e5..52ba135 100644
--- a/tests/suite/table.rs
+++ b/tests/suite/table.rs
@@ -325,7 +325,7 @@ fn table_test_15() {
 - | -
 1 | 2
 "##;
-    let expected = r##"<p>a | b</p>
+    let expected = r##"<p>a | b\</p>
 <ul>
 <li>| - 1 | 2</li>
 </ul>
@@ -333,3 +333,19 @@ fn table_test_15() {
 
     test_markdown_html(original, expected, false, false, false);
 }
+
+#[test]
+fn table_test_16() {
+    let original = r##"a\
+| b | c |
+|---|---|
+| d | e |
+"##;
+    let expected = r##"<p>a\</p>
+<table><thead><tr><th>b</th><th>c</th></tr></thead><tbody>
+<tr><td>d</td><td>e</td></tr>
+</tbody></table>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}