Repository: pulldown-cmark/pulldown-cmark
Pull request #864 (2024-03-13 19:13:21)

Adjust strikethrough flanking rule to better fit Rustdoc Crater run
This is written in response to https://github.com/rust-lang/rust/pull/121659


Patch:
diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs
index f453dc5..1b9352b 100644
--- a/fuzz/src/lib.rs
+++ b/fuzz/src/lib.rs
@@ -15,11 +15,7 @@ use pulldown_cmark::{CodeBlockKind, Event, LinkType, Parser, Tag, TagEnd};
 use quick_xml::escape::unescape;
 use quick_xml::events::Event as XmlEvent;
 use quick_xml::reader::Reader;
-
-fn urldecode(data: &str) -> String {
-    let decoded = urlencoding::decode_binary(data.as_bytes());
-    urlencoding::encode_binary(&decoded[..]).to_string()
-}
+use urlencoding::decode as urldecode;
 
 /// Send Markdown `text` to `pulldown-cmark` and return Markdown
 /// events.
@@ -175,12 +171,11 @@ pub fn xml_to_events(xml: &str) -> anyhow::Result<Vec<Event>> {
                         .into(),
                 )),
                 name @ (b"link" | b"image") => {
-                    let dest_url = tag
+                    let url_encoded_dest_url = tag
                         .try_get_attribute("destination")?
                         .ok_or(anyhow!("Missing destination"))?
-                        .unescape_value()?
-                        .into_owned()
-                        .into();
+                        .unescape_value()?;
+                    let dest_url = urldecode(&url_encoded_dest_url)?.into_owned().into();
                     let title = match tag.try_get_attribute("title")? {
                         Some(title) => title.unescape_value()?.into_owned().into(),
                         None => "".into(),
@@ -318,7 +313,7 @@ pub fn normalize(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
                 ..
             }) => Some(Event::Start(Tag::Link {
                 link_type: LinkType::Inline,
-                dest_url: urldecode(&format!("mailto:{dest_url}")).into(),
+                dest_url: format!("mailto:{dest_url}").into(),
                 title: title.clone(),
                 id: "".into(), // commonmark.js does not record this
             })),
@@ -328,7 +323,7 @@ pub fn normalize(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
                 ..
             }) => Some(Event::Start(Tag::Link {
                 link_type: LinkType::Inline,
-                dest_url: urldecode(&dest_url).into(),
+                dest_url: dest_url.clone(),
                 title: title.clone(),
                 id: "".into(), // commonmark.js does not record this
             })),
@@ -340,7 +335,7 @@ pub fn normalize(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
                 ..
             }) => Some(Event::Start(Tag::Image {
                 link_type: LinkType::Inline,
-                dest_url: urldecode(&dest_url).into(),
+                dest_url: dest_url.clone(),
                 title: title.clone(),
                 id: id.clone(),
             })),
diff --git a/pulldown-cmark/specs/regression.txt b/pulldown-cmark/specs/regression.txt
index daf90c8..5aca680 100644
--- a/pulldown-cmark/specs/regression.txt
+++ b/pulldown-cmark/specs/regression.txt
@@ -2511,34 +2511,3 @@ ISSUE 853
 .
 <p><a href="((()))">linky</a></p>
 ````````````````````````````````
-
-ISSUE 857
-
-```````````````````````````````` example
-><span
-title
->junk
-.
-<blockquote>
-<p>&lt;span
-title
-junk</p>
-</blockquote>
-````````````````````````````````
-
-ISSUE 861
-
-```````````````````````````````` example
- ~~~
-    ~~~
-.
-<pre><code>   ~~~
-</code></pre>
-````````````````````````````````
-
-```````````````````````````````` example
- ~~~
-   ~~~
-.
-<pre><code></code></pre>
-````````````````````````````````
diff --git a/pulldown-cmark/specs/strikethrough.txt b/pulldown-cmark/specs/strikethrough.txt
new file mode 100644
index 0000000..73c8df6
--- /dev/null
+++ b/pulldown-cmark/specs/strikethrough.txt
@@ -0,0 +1,125 @@
+This is an extension of gfm_strikethrough.txt. Some of these tests are also pulled from commonmark-hs.
+
+# Two tildes
+
+Basic strikethrough is between two tildes:
+
+```````````````````````````````` example
+~~This is *stricken out*~~
+.
+<p><del>This is <em>stricken out</em></del></p>
+````````````````````````````````
+
+Backslash escapes:
+
+```````````````````````````````` example
+~~This is \~\~stricken~~
+.
+<p><del>This is ~~stricken</del></p>
+````````````````````````````````
+
+Intraword strikeout:
+
+```````````````````````````````` example
+This~~is~~stricken
+.
+<p>This<del>is</del>stricken</p>
+````````````````````````````````
+
+```````````````````````````````` example
+~~This~~is~~stricken~~
+.
+<p><del>This</del>is<del>stricken</del></p>
+````````````````````````````````
+
+Punctuation is ignored for purposes of determining
+flankingness on two tildes:
+
+```````````````````````````````` example
+Here I strike out an exclamation point~~!~~.
+.
+<p>Here I strike out an exclamation point<del>!</del>.</p>
+````````````````````````````````
+
+# One tilde
+
+One tilde—and this is where we differ from commonmark-hs—is allowed in certain situations:
+
+```````````````````````````````` example
+~This is stricken out~
+.
+<p><del>This is stricken out</del></p>
+````````````````````````````````
+
+Backslash escapes:
+
+```````````````````````````````` example
+~This is \~stricken~
+.
+<p><del>This is ~stricken</del></p>
+````````````````````````````````
+
+Intraword strikeout requires two tildes:
+
+```````````````````````````````` example
+This~is~nothing
+.
+<p>This~is~nothing</p>
+````````````````````````````````
+
+```````````````````````````````` example
+~This~is~nothing~
+.
+<p><del>This~is~nothing</del></p>
+````````````````````````````````
+
+Punctuation is used for purposes of determining
+flankingness:
+
+```````````````````````````````` example
+Here I fail to strike out an exclamation point~!~.
+.
+<p>Here I fail to strike out an exclamation point~!~.</p>
+````````````````````````````````
+
+Tilde runs can't mix.
+
+```````````````````````````````` example
+Here I fail to strike out a tilde ~~~.
+.
+<p>Here I fail to strike out a tilde ~~~.</p>
+````````````````````````````````
+
+```````````````````````````````` example
+Here I fail to match up ~~tildes~.
+.
+<p>Here I fail to match up ~~tildes~.</p>
+````````````````````````````````
+
+```````````````````````````````` example
+Here I fail to match up ~tildes~~.
+.
+<p>Here I fail to match up ~tildes~~.</p>
+````````````````````````````````
+
+Double tildes are allowed to contain single tildes, and the other way around:
+
+```````````````````````````````` example
+~~This ~is stricken.~~
+.
+<p><del>This ~is stricken.</del></p>
+````````````````````````````````
+
+```````````````````````````````` example
+~This ~~is stricken.~
+.
+<p><del>This ~~is stricken.</del></p>
+````````````````````````````````
+
+The first one wins.
+
+```````````````````````````````` example
+~This ~~is stricken~ but this is not~~
+.
+<p><del>This ~~is stricken</del> but this is not~~</p>
+````````````````````````````````
diff --git a/pulldown-cmark/src/firstpass.rs b/pulldown-cmark/src/firstpass.rs
index e15cdd8..35dd6fa 100644
--- a/pulldown-cmark/src/firstpass.rs
+++ b/pulldown-cmark/src/firstpass.rs
@@ -1148,7 +1148,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
             }
             line_start.scan_space(indent);
             let mut close_line_start = line_start.clone();
-            if !close_line_start.scan_space(4 - indent) {
+            if !close_line_start.scan_space(4) {
                 let close_ix = ix + close_line_start.bytes_scanned();
                 if let Some(n) = scan_closing_code_fence(&bytes[close_ix..], fence_ch, n_fence_char)
                 {
@@ -2037,12 +2037,17 @@ fn delim_run_can_open(
         }
     }
     let delim = suffix.chars().next().unwrap();
-    // `*` and `~~` can be intraword, `_` cannot
-    if (delim == '*' || delim == '~') && !is_punctuation(next_char) {
+    // `*` and `~~` can be intraword, `_` and `~` cannot
+    if delim == '*' && !is_punctuation(next_char) {
+        return true;
+    }
+    if delim == '~' && run_len > 1 {
         return true;
     }
-
     let prev_char = s[..ix].chars().last().unwrap();
+    if delim == '~' && prev_char == '~' && !is_punctuation(next_char) {
+        return true;
+    }
 
     prev_char.is_whitespace()
         || is_punctuation(prev_char) && (delim != '\'' || ![']', ')'].contains(&prev_char))
@@ -2079,8 +2084,11 @@ fn delim_run_can_close(
         }
     }
     let delim = suffix.chars().next().unwrap();
-    // `*` and `~~` can be intraword, `_` cannot
-    if (delim == '*' || delim == '~') && !is_punctuation(prev_char) {
+    // `*` and `~~` can be intraword, `_` and `~` cannot
+    if (delim == '*' || (delim == '~' && run_len > 1)) && !is_punctuation(prev_char) {
+        return true;
+    }
+    if delim == '~' && prev_char == '~' {
         return true;
     }
 
diff --git a/pulldown-cmark/src/parse.rs b/pulldown-cmark/src/parse.rs
index 054900e..29331ae 100644
--- a/pulldown-cmark/src/parse.rs
+++ b/pulldown-cmark/src/parse.rs
@@ -1261,6 +1261,9 @@ impl InlineStack {
             .cloned()
             .enumerate()
             .rfind(|(_, el)| {
+                if c == b'~' && run_length != el.run_length {
+                    return false;
+                }
                 el.c == c && (!both && !el.both || (run_length + el.run_length) % 3 != 0 || run_length % 3 == 0)
             });
 
diff --git a/pulldown-cmark/src/scanners.rs b/pulldown-cmark/src/scanners.rs
index 73bf33a..728d233 100644
--- a/pulldown-cmark/src/scanners.rs
+++ b/pulldown-cmark/src/scanners.rs
@@ -915,18 +915,18 @@ fn scan_attribute(
     buffer_ix: &mut usize,
 ) -> Option<usize> {
     ix += scan_attribute_name(&data[ix..])?;
-    let ix_after_attribute = ix;
-    ix = scan_whitespace_with_newline_handler_without_buffer(data, ix, newline_handler)?;
+    let n_whitespace =
+        scan_whitespace_with_newline_handler(data, ix, newline_handler, buffer, buffer_ix)? - ix;
+    ix += n_whitespace;
     if scan_ch(&data[ix..], b'=') == 1 {
-        ix = scan_whitespace_with_newline_handler(data, ix_after_attribute, newline_handler, buffer, buffer_ix)?;
         ix += 1;
         ix = scan_whitespace_with_newline_handler(data, ix, newline_handler, buffer, buffer_ix)?;
         ix = scan_attribute_value(data, ix, newline_handler, buffer, buffer_ix)?;
-        Some(ix)
-    } else {
+    } else if n_whitespace > 0 {
         // Leave whitespace for next attribute.
-        Some(ix_after_attribute)
+        ix -= 1;
     }
+    Some(ix)
 }
 
 /// Scans whitespace and possibly newlines according to the
@@ -962,35 +962,6 @@ fn scan_whitespace_with_newline_handler(
     Some(i)
 }
 
-/// Scans whitespace and possible newlines according to the behavior defined
-/// by the newline handler.
-///
-/// Unlike [`scan_whitespace_with_newline_handler`], this function doesn't
-/// copy skipped data into a buffer. Typically, if this function
-/// returns `Some`, a call to `scan_whitespace_with_newline_handler` will
-/// soon follow.
-fn scan_whitespace_with_newline_handler_without_buffer(
-    data: &[u8],
-    mut i: usize,
-    newline_handler: Option<&dyn Fn(&[u8]) -> usize>,
-) -> Option<usize> {
-    while i < data.len() {
-        if !is_ascii_whitespace(data[i]) {
-            return Some(i);
-        }
-        if let Some(eol_bytes) = scan_eol(&data[i..]) {
-            let handler = newline_handler?;
-            i += eol_bytes;
-            let skipped_bytes = handler(&data[i..]);
-            i += skipped_bytes;
-        } else {
-            i += 1;
-        }
-    }
-
-    Some(i)
-}
-
 /// Returns the index immediately following the attribute value on success.
 fn scan_attribute_value(
     data: &[u8],


Test patch:
diff --git a/pulldown-cmark/tests/suite/mod.rs b/pulldown-cmark/tests/suite/mod.rs
index 932782d..7ee7e64 100644
--- a/pulldown-cmark/tests/suite/mod.rs
+++ b/pulldown-cmark/tests/suite/mod.rs
@@ -13,4 +13,5 @@ mod old_footnotes;
 mod regression;
 mod smart_punct;
 mod spec;
+mod strikethrough;
 mod table;
diff --git a/pulldown-cmark/tests/suite/regression.rs b/pulldown-cmark/tests/suite/regression.rs
index 41f2568..d5e1fad 100644
--- a/pulldown-cmark/tests/suite/regression.rs
+++ b/pulldown-cmark/tests/suite/regression.rs
@@ -3000,42 +3000,3 @@ fn regression_test_189() {
 
     test_markdown_html(original, expected, false, false, false);
 }
-
-#[test]
-fn regression_test_190() {
-    let original = r##"><span
-title
->junk
-"##;
-    let expected = r##"<blockquote>
-<p>&lt;span
-title
-junk</p>
-</blockquote>
-"##;
-
-    test_markdown_html(original, expected, false, false, false);
-}
-
-#[test]
-fn regression_test_191() {
-    let original = r##" ~~~
-    ~~~
-"##;
-    let expected = r##"<pre><code>   ~~~
-</code></pre>
-"##;
-
-    test_markdown_html(original, expected, false, false, false);
-}
-
-#[test]
-fn regression_test_192() {
-    let original = r##" ~~~
-   ~~~
-"##;
-    let expected = r##"<pre><code></code></pre>
-"##;
-
-    test_markdown_html(original, expected, false, false, false);
-}
diff --git a/pulldown-cmark/tests/suite/strikethrough.rs b/pulldown-cmark/tests/suite/strikethrough.rs
new file mode 100644
index 0000000..a8799a3
--- /dev/null
+++ b/pulldown-cmark/tests/suite/strikethrough.rs
@@ -0,0 +1,164 @@
+// This file is auto-generated by the build script
+// Please, do not modify it manually
+
+use super::test_markdown_html;
+
+#[test]
+fn strikethrough_test_1() {
+    let original = r##"~~This is *stricken out*~~
+"##;
+    let expected = r##"<p><del>This is <em>stricken out</em></del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_2() {
+    let original = r##"~~This is \~\~stricken~~
+"##;
+    let expected = r##"<p><del>This is ~~stricken</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_3() {
+    let original = r##"This~~is~~stricken
+"##;
+    let expected = r##"<p>This<del>is</del>stricken</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_4() {
+    let original = r##"~~This~~is~~stricken~~
+"##;
+    let expected = r##"<p><del>This</del>is<del>stricken</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_5() {
+    let original = r##"Here I strike out an exclamation point~~!~~.
+"##;
+    let expected = r##"<p>Here I strike out an exclamation point<del>!</del>.</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_6() {
+    let original = r##"~This is stricken out~
+"##;
+    let expected = r##"<p><del>This is stricken out</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_7() {
+    let original = r##"~This is \~stricken~
+"##;
+    let expected = r##"<p><del>This is ~stricken</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_8() {
+    let original = r##"This~is~nothing
+"##;
+    let expected = r##"<p>This~is~nothing</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_9() {
+    let original = r##"~This~is~nothing~
+"##;
+    let expected = r##"<p><del>This~is~nothing</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_10() {
+    let original = r##"Here I fail to strike out an exclamation point~!~.
+"##;
+    let expected = r##"<p>Here I fail to strike out an exclamation point~!~.</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_11() {
+    let original = r##"Here I fail to strike out a tilde ~~~.
+"##;
+    let expected = r##"<p>Here I fail to strike out a tilde ~~~.</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_12() {
+    let original = r##"Here I fail to match up ~~tildes~.
+"##;
+    let expected = r##"<p>Here I fail to match up ~~tildes~.</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_13() {
+    let original = r##"Here I fail to match up ~tildes~~.
+"##;
+    let expected = r##"<p>Here I fail to match up ~tildes~~.</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_14() {
+    let original = r##"~~This ~is stricken.~~
+"##;
+    let expected = r##"<p><del>This ~is stricken.</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_15() {
+    let original = r##"~This ~~is stricken.~
+"##;
+    let expected = r##"<p><del>This ~~is stricken.</del></p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}
+
+#[test]
+fn strikethrough_test_16() {
+    let original = r##"~This ~~is stricken~ but this is not~~
+"##;
+    let expected = r##"<p><del>This ~~is stricken</del> but this is not~~</p>
+"##;
+
+    test_markdown_html(original, expected, false, false, false);
+}