Repository: raphlinus/pulldown-cmark (rust)

Issue:
Parsing inline code with `"\n "`  loses both newline and space
Events from pulldown-cmark:

```rust
"`\n `f " -> [
  Start(Paragraph)
    Code(Borrowed(""))
    Text(Borrowed("f"))
  End(Paragraph)
]
```

Events from commonmark.js:

```rust
"`\n `f " -> [
  Start(Paragraph)
    Code(Boxed(" "))
    Text(Boxed("f "))
  End(Paragraph)
]
```

XML from [commonmark.js](https://spec.commonmark.org/dingus/?text=%60%0A%20%60f):

```xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE document SYSTEM "CommonMark.dtd">
<document xmlns="http://commonmark.org/xml/1.0">
  <paragraph>
    <code> </code>
    <text>f</text>
  </paragraph>
</document>
```

This might be the same as #655, but here it looks like ``"`\n `"`` is turned into `""`, which means that both the space and the newline got lost.

Pull request description:
Fix parsing of spaces in code
Fixes #661

The changed case in regression.txt actually aligns our behavior with [commonmark.js](https://spec.commonmark.org/dingus/?text=%3E%20%60%0A%3E%20%60&smart=1). Check the "HTML" tab.

This change simplifies everything by having the code span maker read the text directly, instead of deciphering the tree with all the emphasis data and trimmed spaces (that still gets used if it later finds out that MaybeCode isn't code).

Patch:
diff --git a/specs/regression.txt b/specs/regression.txt
index 12e5c03..b936b15 100644
--- a/specs/regression.txt
+++ b/specs/regression.txt
@@ -450,7 +450,7 @@ EMPTY CODE IN BLOCKQUOTE
 > `
 .
 <blockquote>
-<p><code></code></p>
+<p><code> </code></p>
 </blockquote>
 ````````````````````````````````
 
diff --git a/src/parse.rs b/src/parse.rs
index 1678d4b..f6e5049 100644
--- a/src/parse.rs
+++ b/src/parse.rs
@@ -833,97 +833,71 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
     ///
     /// Both `open` and `close` are matching MaybeCode items.
     fn make_code_span(&mut self, open: TreeIndex, close: TreeIndex, preceding_backslash: bool) {
-        let first_ix = self.tree[open].next.unwrap();
         let bytes = self.text.as_bytes();
-        let mut span_start = self.tree[open].item.end;
-        let mut span_end = self.tree[close].item.start;
+        let span_start = self.tree[open].item.end;
+        let span_end = self.tree[close].item.start;
         let mut buf: Option<String> = None;
 
-        // detect all-space sequences, since they are kept as-is as of commonmark 0.29
-        if !bytes[span_start..span_end].iter().all(|&b| b == b' ') {
-            let opening = matches!(bytes[span_start], b' ' | b'\r' | b'\n');
-            let closing = matches!(bytes[span_end - 1], b' ' | b'\r' | b'\n');
-            let drop_enclosing_whitespace = opening && closing;
-
-            if drop_enclosing_whitespace {
-                span_start += 1;
-                if span_start < span_end {
-                    span_end -= 1;
-                }
-            }
-
-            let mut ix = first_ix;
-
-            while ix != close {
-                let next_ix = self.tree[ix].next.unwrap();
-                let end = if next_ix == close {
-                    span_end
-                } else {
-                    self.tree[next_ix].item.start
-                };
-                if let ItemBody::HardBreak(_) | ItemBody::SoftBreak = self.tree[ix].item.body {
-                    if drop_enclosing_whitespace {
-                        // check whether break should be ignored
-                        if ix == first_ix {
-                            ix = next_ix;
-                            span_start = min(span_end, self.tree[ix].item.start);
-                            continue;
-                        } else if next_ix == close && ix > first_ix {
-                            break;
-                        }
-                    }
-
-                    let end = bytes[self.tree[ix].item.start..]
-                        .iter()
-                        .position(|&b| b == b'\r' || b == b'\n')
-                        .unwrap()
-                        + self.tree[ix].item.start;
-                    if let Some(ref mut buf) = buf {
-                        buf.push_str(&self.text[self.tree[ix].item.start..end]);
-                        buf.push(' ');
-                    } else {
-                        let mut new_buf = String::with_capacity(span_end - span_start);
-                        new_buf.push_str(&self.text[span_start..end]);
-                        new_buf.push(' ');
-                        buf = Some(new_buf);
-                    }
-                } else if self.tree.is_in_table()
-                    && self.text[self.tree[ix].item.start..end].contains("|")
-                {
-                    for (i, c) in bytes[self.tree[ix].item.start..end]
-                        .into_iter()
-                        .copied()
-                        .enumerate()
-                    {
-                        if c != b'|' {
-                            continue;
-                        }
-                        let pipe_position = self.tree[ix].item.start + i;
-                        let buf = if let Some(ref mut buf) = buf {
-                            buf.push_str(&self.text[self.tree[ix].item.start..pipe_position]);
-                            buf
-                        } else {
-                            let mut new_buf = String::with_capacity(pipe_position - span_start);
-                            new_buf.push_str(&self.text[span_start..pipe_position]);
-                            buf.insert(new_buf)
-                        };
-                        let ends_in_backslash = buf.as_bytes().last() == Some(&b'\\');
-                        let trim = if ends_in_backslash { 1 } else { 0 };
-                        buf.truncate(buf.len() - trim);
-                        buf.push_str(&self.text[pipe_position..end]);
-                    }
-                } else if let Some(ref mut buf) = buf {
-                    buf.push_str(&self.text[self.tree[ix].item.start..end]);
-                }
-                ix = next_ix;
+        let mut start_ix = span_start;
+        let mut ix = span_start;
+        while ix < span_end {
+            let c = bytes[ix];
+            if c == b'\r' || c == b'\n' {
+                let buf = buf.get_or_insert_with(|| String::with_capacity(ix + 1 - span_start));
+                buf.extend(self.text[start_ix..ix].chars());
+                buf.push(' ');
+                ix += 1;
+                let mut line_start = LineStart::new(&bytes[ix..]);
+                let _ = scan_containers(
+                    &self.tree,
+                    &mut line_start,
+                    self.options.has_gfm_footnotes(),
+                );
+                ix += line_start.bytes_scanned();
+                start_ix = ix;
+            } else if c == b'\\' && bytes.get(ix + 1) == Some(&b'|') && self.tree.is_in_table() {
+                let buf = buf.get_or_insert_with(|| String::with_capacity(ix + 1 - span_start));
+                buf.extend(self.text[start_ix..ix].chars());
+                buf.push('|');
+                ix += 2;
+                start_ix = ix;
+            } else {
+                ix += 1;
             }
         }
+        
+        let (opening, closing, all_spaces) = {
+            let s = if let Some(buf) = &mut buf {
+                buf.extend(self.text[start_ix..span_end].chars());
+                &buf[..]
+            } else {
+                &self.text[span_start..span_end]
+            };
+            (
+                s.as_bytes().first() == Some(&b' '),
+                s.as_bytes().last() == Some(&b' '),
+                s.bytes().all(|b| b == b' ')
+            )
+        };
 
-        let cow = if let Some(buf) = buf {
-            buf.into()
+        let cow: CowStr<'input> = if !all_spaces && opening && closing {
+            if let Some(mut buf) = buf {
+                buf.remove(0);
+                buf.pop();
+                buf.into()
+            } else {
+                let lo = span_start + 1;
+                let hi = (span_end - 1).max(lo);
+                self.text[lo..hi].into()
+            }
         } else {
-            self.text[span_start..span_end].into()
+            if let Some(buf) = buf {
+                buf.into()
+            } else {
+                self.text[span_start..span_end].into()
+            }
         };
+
         if preceding_backslash {
             self.tree[open].item.body = ItemBody::Text;
             self.tree[open].item.end = self.tree[open].item.start + 1;


Test patch:
diff --git a/tests/html.rs b/tests/html.rs
index 5b29e3d..5e2eddb 100644
--- a/tests/html.rs
+++ b/tests/html.rs
@@ -249,3 +249,37 @@ fn html_test_broken_callback() {
 
     assert_eq!(expected, s);
 }
+
+#[test]
+fn newline_in_code() {
+    let originals = ["`\n `x", "` \n`x"];
+    let expected = "<p><code>  </code>x</p>\n";
+
+    for original in originals {
+        let mut s = String::new();
+        html::push_html(&mut s, Parser::new(&original));
+        assert_eq!(expected, s);
+    }
+}
+
+#[test]
+fn newline_start_end_of_code() {
+    let original = "`\nx\n`x";
+    let expected = "<p><code>x</code>x</p>\n";
+
+    let mut s = String::new();
+    html::push_html(&mut s, Parser::new(&original));
+    assert_eq!(expected, s);
+}
+
+#[test]
+fn newline_within_code() {
+    let originals = ["`\nx \ny\n`x", "`x \ny`x", "`x\n y`x"];
+    let expected = "<p><code>x  y</code>x</p>\n";
+
+    for original in originals {
+        let mut s = String::new();
+        html::push_html(&mut s, Parser::new(&original));
+        assert_eq!(expected, s);
+    }
+}
diff --git a/tests/suite/regression.rs b/tests/suite/regression.rs
index 1cb8068..e2a4be9 100644
--- a/tests/suite/regression.rs
+++ b/tests/suite/regression.rs
@@ -530,7 +530,7 @@ fn regression_test_39() {
 > `
 "##;
     let expected = r##"<blockquote>
-<p><code></code></p>
+<p><code> </code></p>
 </blockquote>
 "##;
 
@@ -2073,16 +2073,22 @@ foo)
 <p><a href="foo">link</a></p>
 </blockquote>
 "##;
-  
+
     test_markdown_html(original, expected, false, false, false);
 }
-  
+
 #[test]
 fn regression_test_131() {
-    let original = r##"[linkme]: foo
-    - baz
+    let original = r##"[link](foo
+)
+
+> [link](foo
+> )
 "##;
-    let expected = r##"<p>- baz</p>
+    let expected = r##"<p><a href="foo">link</a></p>
+<blockquote>
+<p><a href="foo">link</a></p>
+</blockquote>
 "##;
 
     test_markdown_html(original, expected, false, false, false);
@@ -2091,22 +2097,56 @@ fn regression_test_131() {
 #[test]
 fn regression_test_132() {
     let original = r##"[link](foo
+"bar"
 )
 
 > [link](foo
+> "bar"
 > )
 "##;
-    let expected = r##"<p><a href="foo">link</a></p>
+    let expected = r##"<p><a href="foo" title="bar">link</a></p>
 <blockquote>
-<p><a href="foo">link</a></p>
+<p><a href="foo" title="bar">link</a></p>
 </blockquote>
 "##;
- 
-  test_markdown_html(original, expected, false, false, false);
+
+    test_markdown_html(original, expected, false, false, false);
 }
-  
-  #[test]
+
+#[test]
 fn regression_test_133() {
+    let original = r##"[link](	
+	foo	
+	"bar"	
+	)
+
+> [link](	
+> 	foo	
+> 	"bar"	
+> 	)
+"##;
+    let expected = r##"<p><a href="foo" title="bar">link</a></p>
+<blockquote>
+<p><a href="foo" title="bar">link</a></p>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_134() {
+    let original = r##"[linkme]: foo
+    - baz
+"##;
+    let expected = r##"<p>- baz</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_135() {
     let original = r##"[linkme-3]:
    [^foo]:
 
@@ -2124,26 +2164,7 @@ GFM footnotes can interrupt link defs if they have three spaces, but not four.
 }
 
 #[test]
-fn regression_test_134() {
-    let original = r##"[link](foo
-"bar"
-)
-
-> [link](foo
-> "bar"
-> )
-"##;
-    let expected = r##"<p><a href="foo" title="bar">link</a></p>
-<blockquote>
-<p><a href="foo" title="bar">link</a></p>
-</blockquote>
-"##;
-
-    test_markdown_html(original, expected, false, false, false);
-}
-
-  #[test]
-fn regression_test_135() {
+fn regression_test_136() {
     let original = r##"[linkme-3]:
    ===
 
@@ -2160,7 +2181,7 @@ Setext heading can interrupt link def if it has three spaces, but not four.
 }
 
 #[test]
-fn regression_test_136() {
+fn regression_test_137() {
     let original = r##"[linkme-3]: a
    - a