Repository: pulldown-cmark/pulldown-cmark
Pull request #969 (2024-10-08 21:22:35)

Respect line starts when trimming header endings
This prevents it from trying to trim whitespace that's part of a block quote or list marker.

Fixes https://github.com/pulldown-cmark/pulldown-cmark/issues/963

This is an alternative fix to https://github.com/pulldown-cmark/pulldown-cmark/pull/968 that handles a few corner cases better. As @rhysd figured out the root cause, the problem is that the code stripped a space that was actually part of the block quote marker, attempting to trim the text node beyond its starting point.


Patch:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index e6426a6..8534da5 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -19,9 +19,6 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install Rust
       run: rustup default ${{ matrix.rust-version }}
-    - name: Cargo fmt
-      if: ${{ matrix.rust-version == 'stable' }}
-      run: cargo fmt --check
     - name: Cargo build
       if: ${{ matrix.rust-version == '1.71.1' }}
       run: cargo build --verbose
diff --git a/bench/benches/lib.rs b/bench/benches/lib.rs
index 288af79..854d5e3 100644
--- a/bench/benches/lib.rs
+++ b/bench/benches/lib.rs
@@ -5,10 +5,8 @@ mod to_html {
     use pulldown_cmark::{html, Options, Parser};
 
     pub fn pathological_missing_table_cells(c: &mut Criterion) {
-        let mut group = c.benchmark_group(
-            "    pub fn pathological_missing_table_cells(c: &mut Criterion) {
-            ",
-        );
+        let mut group = c.benchmark_group("    pub fn pathological_missing_table_cells(c: &mut Criterion) {
+            ");
         let mut buf = String::new();
         for i in 1..20 {
             buf.clear();
@@ -26,10 +24,8 @@ mod to_html {
     }
 
     pub fn pathological_link_def(c: &mut Criterion) {
-        let mut group = c.benchmark_group(
-            "    pub fn pathological_link_def(c: &mut Criterion) {
-            ",
-        );
+        let mut group = c.benchmark_group("    pub fn pathological_link_def(c: &mut Criterion) {
+            ");
         let mut buf = String::new();
         for i in 1..20 {
             buf.clear();
diff --git a/bench/src/lib.rs b/bench/src/lib.rs
index 8b13789..e69de29 100644
--- a/bench/src/lib.rs
+++ b/bench/src/lib.rs
@@ -1,1 +0,0 @@
-
diff --git a/dos-fuzzer/src/scoring.rs b/dos-fuzzer/src/scoring.rs
index 48d1eb3..833785e 100644
--- a/dos-fuzzer/src/scoring.rs
+++ b/dos-fuzzer/src/scoring.rs
@@ -12,9 +12,7 @@ pub fn pearson_correlation(time_samples: &[(f64, f64)]) -> (f64, bool) {
     vec.extend(time_samples.iter().cloned().map(|(x, y)| [x, y]));
     let time_samples = Array2::from(vec);
     let time_samples = time_samples.t();
-    let corr = time_samples
-        .pearson_correlation()
-        .expect("no time samples given")[[1, 0]];
+    let corr = time_samples.pearson_correlation().expect("no time samples given")[[1, 0]];
     (corr, corr < super::ACCEPTANCE_CORRELATION)
 }
 
diff --git a/pulldown-cmark/specs/regression.txt b/pulldown-cmark/specs/regression.txt
index 44442e6..f7ccb68 100644
--- a/pulldown-cmark/specs/regression.txt
+++ b/pulldown-cmark/specs/regression.txt
@@ -2726,3 +2726,59 @@ Some preamble <code>foobar_raz</code>, not <code>barfoo_raz</code></dt>
 stuff](https://example.com)</li>
 </ul>
 ````````````````````````````````
+
+ISSUE #963
+
+```````````````````````````````` example
+foo
+{.class}
+===
+
+> foo
+> {.class}
+> ===
+>
+> > foo
+> > {.class}
+> > ===
+
+* > foo
+  > {.class}
+  > ===
+
+> foo
+>→{.class}
+>→===
+.
+<h1 class="class">foo
+</h1>
+<blockquote>
+<h1 class="class">foo
+</h1>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+</blockquote>
+<ul>
+<li>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+</li>
+</ul>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+````````````````````````````````
+
+```````````````````````````````` example
+the trailing space after the &gt; should be stripped
+    > {.bar}
+===
+.
+<h1 class="bar">the trailing space after the &gt; should be stripped
+&gt;</h1>
+````````````````````````````````
diff --git a/pulldown-cmark/src/firstpass.rs b/pulldown-cmark/src/firstpass.rs
index 0038d14..593ce74 100644
--- a/pulldown-cmark/src/firstpass.rs
+++ b/pulldown-cmark/src/firstpass.rs
@@ -142,8 +142,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                 }
             }
             let container_start = start_ix + line_start.bytes_scanned();
-            if let Some((ch, index, indent)) = line_start.scan_list_marker_with_indent(outer_indent)
-            {
+            if let Some((ch, index, indent)) = line_start.scan_list_marker_with_indent(outer_indent) {
                 let after_marker_index = start_ix + line_start.bytes_scanned();
                 self.continue_list(container_start - outer_indent, ch, index);
                 self.tree.append(Item {
@@ -195,8 +194,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                 })
                 .and_then(|item| {
                     Some((
-                        line_start
-                            .scan_definition_list_definition_marker_with_indent(outer_indent)?,
+                        line_start.scan_definition_list_definition_marker_with_indent(outer_indent)?,
                         item.0,
                         item.1,
                     ))
@@ -586,12 +584,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         let body = if let Some(ItemBody::DefinitionList(_)) =
             self.tree.peek_up().map(|idx| self.tree[idx].item.body)
         {
-            if self.tree.cur().map_or(true, |idx| {
-                matches!(
-                    &self.tree[idx].item.body,
-                    ItemBody::DefinitionListDefinition(..)
-                )
-            }) {
+            if self.tree.cur().map_or(true, |idx| matches!(&self.tree[idx].item.body, ItemBody::DefinitionListDefinition(..))) {
                 // blank lines between the previous definition and this one don't count
                 self.last_line_blank = false;
                 ItemBody::MaybeDefinitionListTitle
@@ -731,8 +724,30 @@ impl<'a, 'b> FirstPass<'a, 'b> {
             let new_end = if has_trailing_content {
                 content_end
             } else {
-                let trailing_ws =
-                    scan_rev_while(&bytes[header_start..content_end], is_ascii_whitespace_no_nl);
+                let mut last_line_start = header_start;
+                if attrs.is_some() {
+                    loop {
+                        let next_line_start =
+                            last_line_start + scan_nextline(&bytes[last_line_start..content_end]);
+                        if next_line_start >= content_end {
+                            break;
+                        }
+                        let mut line_start = LineStart::new(&bytes[next_line_start..content_end]);
+                        if scan_containers(
+                            &self.tree,
+                            &mut line_start,
+                            self.options.has_gfm_footnotes(),
+                        ) != self.tree.spine_len()
+                        {
+                            break;
+                        }
+                        last_line_start = next_line_start + line_start.bytes_scanned();
+                    }
+                }
+                let trailing_ws = scan_rev_while(
+                    &bytes[last_line_start..content_end],
+                    is_ascii_whitespace_no_nl,
+                );
                 content_end - trailing_ws
             };
 
diff --git a/pulldown-cmark/src/lib.rs b/pulldown-cmark/src/lib.rs
index 6702f82..d7e045e 100644
--- a/pulldown-cmark/src/lib.rs
+++ b/pulldown-cmark/src/lib.rs
@@ -120,13 +120,6 @@ impl<'a> CodeBlockKind<'a> {
     pub fn is_fenced(&self) -> bool {
         matches!(*self, CodeBlockKind::Fenced(_))
     }
-
-    pub fn into_static(self) -> CodeBlockKind<'static> {
-        match self {
-            CodeBlockKind::Indented => CodeBlockKind::Indented,
-            CodeBlockKind::Fenced(s) => CodeBlockKind::Fenced(s.into_static()),
-        }
-    }
 }
 
 /// BlockQuote kind (Note, Tip, Important, Warning, Caution).
@@ -251,65 +244,6 @@ impl<'a> Tag<'a> {
             Tag::DefinitionListDefinition => TagEnd::DefinitionListDefinition,
         }
     }
-
-    pub fn into_static(self) -> Tag<'static> {
-        match self {
-            Tag::Paragraph => Tag::Paragraph,
-            Tag::Heading {
-                level,
-                id,
-                classes,
-                attrs,
-            } => Tag::Heading {
-                level,
-                id: id.map(|s| s.into_static()),
-                classes: classes.into_iter().map(|s| s.into_static()).collect(),
-                attrs: attrs
-                    .into_iter()
-                    .map(|(k, v)| (k.into_static(), v.map(|s| s.into_static())))
-                    .collect(),
-            },
-            Tag::BlockQuote(k) => Tag::BlockQuote(k),
-            Tag::CodeBlock(kb) => Tag::CodeBlock(kb.into_static()),
-            Tag::HtmlBlock => Tag::HtmlBlock,
-            Tag::List(v) => Tag::List(v),
-            Tag::Item => Tag::Item,
-            Tag::FootnoteDefinition(a) => Tag::FootnoteDefinition(a.into_static()),
-            Tag::Table(v) => Tag::Table(v),
-            Tag::TableHead => Tag::TableHead,
-            Tag::TableRow => Tag::TableRow,
-            Tag::TableCell => Tag::TableCell,
-            Tag::Emphasis => Tag::Emphasis,
-            Tag::Strong => Tag::Strong,
-            Tag::Strikethrough => Tag::Strikethrough,
-            Tag::Link {
-                link_type,
-                dest_url,
-                title,
-                id,
-            } => Tag::Link {
-                link_type,
-                dest_url: dest_url.into_static(),
-                title: title.into_static(),
-                id: id.into_static(),
-            },
-            Tag::Image {
-                link_type,
-                dest_url,
-                title,
-                id,
-            } => Tag::Image {
-                link_type,
-                dest_url: dest_url.into_static(),
-                title: title.into_static(),
-                id: id.into_static(),
-            },
-            Tag::MetadataBlock(v) => Tag::MetadataBlock(v),
-            Tag::DefinitionList => Tag::DefinitionList,
-            Tag::DefinitionListTitle => Tag::DefinitionListTitle,
-            Tag::DefinitionListDefinition => Tag::DefinitionListDefinition,
-        }
-    }
 }
 
 /// The end of a `Tag`.
@@ -486,26 +420,6 @@ pub enum Event<'a> {
     TaskListMarker(bool),
 }
 
-impl<'a> Event<'a> {
-    pub fn into_static(self) -> Event<'static> {
-        match self {
-            Event::Start(t) => Event::Start(t.into_static()),
-            Event::End(e) => Event::End(e),
-            Event::Text(s) => Event::Text(s.into_static()),
-            Event::Code(s) => Event::Code(s.into_static()),
-            Event::InlineMath(s) => Event::InlineMath(s.into_static()),
-            Event::DisplayMath(s) => Event::DisplayMath(s.into_static()),
-            Event::Html(s) => Event::Html(s.into_static()),
-            Event::InlineHtml(s) => Event::InlineHtml(s.into_static()),
-            Event::FootnoteReference(s) => Event::FootnoteReference(s.into_static()),
-            Event::SoftBreak => Event::SoftBreak,
-            Event::HardBreak => Event::HardBreak,
-            Event::Rule => Event::Rule,
-            Event::TaskListMarker(b) => Event::TaskListMarker(b),
-        }
-    }
-}
-
 /// Table column text alignment.
 #[derive(Copy, Clone, Debug, PartialEq)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
diff --git a/pulldown-cmark/src/linklabel.rs b/pulldown-cmark/src/linklabel.rs
index e64859a..e6c6fb6 100644
--- a/pulldown-cmark/src/linklabel.rs
+++ b/pulldown-cmark/src/linklabel.rs
@@ -22,7 +22,7 @@
 
 use unicase::UniCase;
 
-use crate::scanners::{is_ascii_punctuation, is_ascii_whitespace, scan_eol};
+use crate::scanners::{is_ascii_whitespace, scan_eol, is_ascii_punctuation};
 use crate::strings::CowStr;
 
 #[derive(Debug)]
@@ -134,16 +134,10 @@ pub(crate) fn scan_link_label_rest<'t>(
             text[..ix].trim_matches(asciiws).into()
         } else {
             label.push_str(&text[mark..ix]);
-            while matches!(
-                label.as_bytes().last(),
-                Some(&b' ' | &b'\r' | &b'\n' | &b'\t')
-            ) {
+            while matches!(label.as_bytes().last(), Some(&b' ' | &b'\r' | &b'\n' | &b'\t')) {
                 label.pop();
             }
-            while matches!(
-                label.as_bytes().first(),
-                Some(&b' ' | &b'\r' | &b'\n' | &b'\t')
-            ) {
+            while matches!(label.as_bytes().first(), Some(&b' ' | &b'\r' | &b'\n' | &b'\t')) {
                 label.remove(0);
             }
             label.into()
diff --git a/pulldown-cmark/src/parse.rs b/pulldown-cmark/src/parse.rs
index 38de1b6..161dc62 100644
--- a/pulldown-cmark/src/parse.rs
+++ b/pulldown-cmark/src/parse.rs
@@ -471,8 +471,7 @@ impl<'input, F: BrokenLinkCallback<'input>> Parser<'input, F> {
                     let result = if self.math_delims.is_populated() {
                         // we have previously scanned all math environment delimiters,
                         // so we can reuse that work
-                        self.math_delims
-                            .find(&self.tree, cur_ix, is_display, brace_context)
+                        self.math_delims.find(&self.tree, cur_ix, is_display, brace_context)
                     } else {
                         // we haven't previously scanned all math delimiters,
                         // so walk the AST
@@ -1688,16 +1687,6 @@ pub struct LinkDef<'a> {
     pub span: Range<usize>,
 }
 
-impl<'a> LinkDef<'a> {
-    pub fn into_static(self) -> LinkDef<'static> {
-        LinkDef {
-            dest: self.dest.into_static(),
-            title: self.title.map(|s| s.into_static()),
-            span: self.span,
-        }
-    }
-}
-
 /// Contains the destination URL, title and source span of a reference definition.
 #[derive(Clone, Debug)]
 pub struct FootnoteDef {
diff --git a/pulldown-cmark/src/scanners.rs b/pulldown-cmark/src/scanners.rs
index ec2e5ec..9bfcf80 100644
--- a/pulldown-cmark/src/scanners.rs
+++ b/pulldown-cmark/src/scanners.rs
@@ -279,10 +279,7 @@ impl<'a> LineStart<'a> {
     ///
     /// Return value is the amount of indentation, or `None` if it's not a
     /// definition list marker.
-    pub(crate) fn scan_definition_list_definition_marker_with_indent(
-        &mut self,
-        indent: usize,
-    ) -> Option<usize> {
+    pub(crate) fn scan_definition_list_definition_marker_with_indent(&mut self, indent: usize) -> Option<usize> {
         let save = self.clone();
         if self.scan_ch(b':') {
             let remaining = 4 - (indent + 1);
@@ -298,10 +295,7 @@ impl<'a> LineStart<'a> {
     /// Return value is the character, the start index, and the indent in spaces.
     /// For ordered list markers, the character will be one of b'.' or b')'. For
     /// bullet list markers, it will be one of b'-', b'+', or b'*'.
-    pub(crate) fn scan_list_marker_with_indent(
-        &mut self,
-        indent: usize,
-    ) -> Option<(u8, u64, usize)> {
+    pub(crate) fn scan_list_marker_with_indent(&mut self, indent: usize) -> Option<(u8, u64, usize)> {
         let save = self.clone();
         if self.ix < self.bytes.len() {
             let c = self.bytes[self.ix];
@@ -1014,13 +1008,7 @@ fn scan_attribute(
     let ix_after_attribute = ix;
     ix = scan_whitespace_with_newline_handler_without_buffer(data, ix, newline_handler)?;
     if scan_ch(&data[ix..], b'=') == 1 {
-        ix = scan_whitespace_with_newline_handler(
-            data,
-            ix_after_attribute,
-            newline_handler,
-            buffer,
-            buffer_ix,
-        )?;
+        ix = scan_whitespace_with_newline_handler(data, ix_after_attribute, newline_handler, buffer, buffer_ix)?;
         ix += 1;
         ix = scan_whitespace_with_newline_handler(data, ix, newline_handler, buffer, buffer_ix)?;
         ix = scan_attribute_value(data, ix, newline_handler, buffer, buffer_ix)?;
diff --git a/pulldown-cmark/src/strings.rs b/pulldown-cmark/src/strings.rs
index a4b13f9..6296435 100644
--- a/pulldown-cmark/src/strings.rs
+++ b/pulldown-cmark/src/strings.rs
@@ -260,17 +260,6 @@ impl<'a> CowStr<'a> {
             CowStr::Inlined(s) => s.deref().to_owned(),
         }
     }
-
-    pub fn into_static(self) -> CowStr<'static> {
-        match self {
-            CowStr::Boxed(b) => CowStr::Boxed(b),
-            CowStr::Borrowed(b) => match InlineStr::try_from(b) {
-                Ok(inline) => CowStr::Inlined(inline),
-                Err(_) => CowStr::Boxed(b.into()),
-            },
-            CowStr::Inlined(s) => CowStr::Inlined(s),
-        }
-    }
 }
 
 impl<'a> fmt::Display for CowStr<'a> {


Test patch:
diff --git a/pulldown-cmark/tests/html.rs b/pulldown-cmark/tests/html.rs
index e8c3bc6..e6c739b 100644
--- a/pulldown-cmark/tests/html.rs
+++ b/pulldown-cmark/tests/html.rs
@@ -330,13 +330,7 @@ fn trim_space_before_soft_break() {
 #[test]
 fn issue_819() {
     let original = [
-        "# \\",
-        "# \\\n",
-        "# \\\n\n",
-        "# \\\r\n",
-        "# \\\r\n\r\n",
-        "# \\\n\r\n",
-        "# \\\r\n\n",
+        "# \\", "# \\\n", "# \\\n\n", "# \\\r\n", "# \\\r\n\r\n", "# \\\n\r\n", "# \\\r\n\n"
     ];
     let expected = "<h1>\\</h1>";
 
diff --git a/pulldown-cmark/tests/lib.rs b/pulldown-cmark/tests/lib.rs
index 6ca5c59..e34cb2a 100644
--- a/pulldown-cmark/tests/lib.rs
+++ b/pulldown-cmark/tests/lib.rs
@@ -2,7 +2,6 @@
 
 use pulldown_cmark::{Options, Parser};
 
-#[rustfmt::skip]
 mod suite;
 
 #[inline(never)]
diff --git a/pulldown-cmark/tests/suite/regression.rs b/pulldown-cmark/tests/suite/regression.rs
index 9a8b1aa..70229e5 100644
--- a/pulldown-cmark/tests/suite/regression.rs
+++ b/pulldown-cmark/tests/suite/regression.rs
@@ -3253,3 +3253,65 @@ stuff](https://example.com)</li>
 
     test_markdown_html(original, expected, false, false, false);
 }
+
+#[test]
+fn regression_test_206() {
+    let original = r##"foo
+{.class}
+===
+
+> foo
+> {.class}
+> ===
+>
+> > foo
+> > {.class}
+> > ===
+
+* > foo
+  > {.class}
+  > ===
+
+> foo
+>	{.class}
+>	===
+"##;
+    let expected = r##"<h1 class="class">foo
+</h1>
+<blockquote>
+<h1 class="class">foo
+</h1>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+</blockquote>
+<ul>
+<li>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+</li>
+</ul>
+<blockquote>
+<h1 class="class">foo
+</h1>
+</blockquote>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn regression_test_207() {
+    let original = r##"the trailing space after the &gt; should be stripped
+    > {.bar}
+===
+"##;
+    let expected = r##"<h1 class="bar">the trailing space after the &gt; should be stripped
+&gt;</h1>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}