Repository: raphlinus/pulldown-cmark (rust)

Issue:
Whitespace before EOL sometimes incorrectly preserved
In the CommonMark spec, 4.8 Paragraphs:

> The paragraph’s raw content is formed by concatenating the lines and **removing** initial and **final spaces or tabs**.

and in 6.8 Soft line breaks:

> Spaces at the end of the line and beginning of the next line are removed:

Some test cases for these (atop `tests/html.rs`):

```rust
// fails
#[test]
fn trim_space_and_tab_at_end_of_paragraph() {
    let original = "one\ntwo \t";
    let expected = "<p>one\ntwo</p>\n";

    let mut s = String::new();
    html::push_html(&mut s, Parser::new(&original));
    assert_eq!(expected, s);
}

// passes
#[test]
fn trim_space_tab_nl_at_end_of_paragraph() {
    let original = "one\ntwo \t\n";
    let expected = "<p>one\ntwo</p>\n";

    let mut s = String::new();
    html::push_html(&mut s, Parser::new(&original));
    assert_eq!(expected, s);
}

// fails
#[test]
fn trim_space_nl_at_end_of_paragraph() {
    let original = "one\ntwo \n";
    let expected = "<p>one\ntwo</p>\n";

    let mut s = String::new();
    html::push_html(&mut s, Parser::new(&original));
    assert_eq!(expected, s);
}

// fails
#[test]
fn trim_space_before_soft_break() {
    let original = "one \ntwo";
    let expected = "<p>one\ntwo</p>\n";

    let mut s = String::new();
    html::push_html(&mut s, Parser::new(&original));
    assert_eq!(expected, s);
}
```

I tried fixing this, but it didn't turn out so simple. `trim_space_tab_nl_at_end_of_paragraph` seems to pass because hard breaks are correctly trimmed out. However, no hard break is recognized if the input doesn't have a trailing newline. So that case will at least require some kind of special handling or modification to `parse_line`.

Pull request description:
Trim spaces off end of parsed lines
Fixes #715

Patch:
diff --git a/src/firstpass.rs b/src/firstpass.rs
index 2c523e6..fcec952 100644
--- a/src/firstpass.rs
+++ b/src/firstpass.rs
@@ -333,11 +333,6 @@ impl<'a, 'b> FirstPass<'a, 'b> {
             self.tree.push();
             let (next_ix, _brk) = self.parse_line(ix, None, TableParseMode::Active);
 
-            if let Some(cur_ix) = self.tree.cur() {
-                let trailing_whitespace = scan_rev_while(&bytes[..next_ix], is_ascii_whitespace);
-                self.tree[cur_ix].item.end -= trailing_whitespace;
-            }
-
             self.tree[cell_ix].item.end = next_ix;
             self.tree.pop();
 
@@ -644,7 +639,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                         );
                     }
 
-                    self.tree.append_text(begin_text, ix);
+                    self.tree.append_text(begin_text, ix - trailing_whitespace);
                     LoopInstruction::BreakAtWith(
                         end_ix,
                         Some(Item {
@@ -854,8 +849,10 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         });
 
         if brk.is_none() {
+            let trailing_whitespace =
+                scan_rev_while(&bytes[begin_text..final_ix], is_ascii_whitespace_no_nl);
             // need to close text at eof
-            self.tree.append_text(begin_text, final_ix);
+            self.tree.append_text(begin_text, final_ix - trailing_whitespace);
         }
         (final_ix, brk)
     }


Test patch:
diff --git a/tests/html.rs b/tests/html.rs
index 5e2eddb..d95d0da 100644
--- a/tests/html.rs
+++ b/tests/html.rs
@@ -272,6 +272,18 @@ fn newline_start_end_of_code() {
     assert_eq!(expected, s);
 }
 
+// https://github.com/raphlinus/pulldown-cmark/issues/715
+
+#[test]
+fn trim_space_and_tab_at_end_of_paragraph() {
+    let original = "one\ntwo \t";
+    let expected = "<p>one\ntwo</p>\n";
+
+    let mut s = String::new();
+    html::push_html(&mut s, Parser::new(&original));
+    assert_eq!(expected, s);
+}
+
 #[test]
 fn newline_within_code() {
     let originals = ["`\nx \ny\n`x", "`x \ny`x", "`x\n y`x"];
@@ -283,3 +295,33 @@ fn newline_within_code() {
         assert_eq!(expected, s);
     }
 }
+
+#[test]
+fn trim_space_tab_nl_at_end_of_paragraph() {
+    let original = "one\ntwo \t\n";
+    let expected = "<p>one\ntwo</p>\n";
+
+    let mut s = String::new();
+    html::push_html(&mut s, Parser::new(&original));
+    assert_eq!(expected, s);
+}
+
+#[test]
+fn trim_space_nl_at_end_of_paragraph() {
+    let original = "one\ntwo \n";
+    let expected = "<p>one\ntwo</p>\n";
+
+    let mut s = String::new();
+    html::push_html(&mut s, Parser::new(&original));
+    assert_eq!(expected, s);
+}
+
+#[test]
+fn trim_space_before_soft_break() {
+    let original = "one \ntwo";
+    let expected = "<p>one\ntwo</p>\n";
+
+    let mut s = String::new();
+    html::push_html(&mut s, Parser::new(&original));
+    assert_eq!(expected, s);
+}