Repository: pulldown-cmark/pulldown-cmark (rust)

Issue:
Ordered or unordered list marker following GitHub Flavored Markdown Task List is parsed as a list instead of text
When enabling [GitHub Flavored Markdown Task Lists](https://github.github.com/gfm/#task-list-items-extension-) with `-L` on the CLI or with `Options::ENABLE_TASKLISTS` in code the parser misinterprets text that looks like a list marker immediately following the task list. What's also odd is that the task list becomes a child of this misinterpreted list even though it comes before it in the source text.

Given the following input:

```markdown
- [x] * some text
```

running `cargo run -- -e -L` produces these events:

```
0..18: Start(List(None))
0..18: Start(Item)
5..18: Start(List(None))
5..18: Start(Item)
2..5: TaskListMarker(true)
8..17: Text(Borrowed("some text"))
5..18: End(Item)
5..18: End(List(false))
0..18: End(Item)
0..18: End(List(false))
EOF
```

If I manually escape the the list marker the parser produces the events I'd expect, which is also how [pandoc interprets the unescaped text](https://pandoc.org/try/?params=%7B%22text%22%3A%22-+%5Bx%5D+*+some+text%22%2C%22to%22%3A%22html5%22%2C%22from%22%3A%22markdown_github%22%2C%22standalone%22%3Afalse%2C%22embed-resources%22%3Afalse%2C%22table-of-contents%22%3Afalse%2C%22number-sections%22%3Afalse%2C%22citeproc%22%3Afalse%2C%22html-math-method%22%3A%22plain%22%2C%22wrap%22%3A%22auto%22%2C%22highlight-style%22%3Anull%2C%22files%22%3A%7B%7D%2C%22template%22%3Anull%7D).

```markdown
- [x] \* some text
```

Again, running with `cargo run -- -e -L`:
```
0..19: Start(List(None))
0..19: Start(Item)
2..5: TaskListMarker(true)
7..18: Text(Borrowed("* some text"))
0..19: End(Item)
0..19: End(List(false))
EOF
```

This is also an issue with text that looks like an ordered list:

```markdown
- [ ] 3. some text
```

```
0..19: Start(List(None))
0..19: Start(Item)
5..19: Start(List(Some(3)))
5..19: Start(Item)
2..5: TaskListMarker(false)
9..18: Text(Borrowed("some text"))
5..19: End(Item)
5..19: End(List(true))
0..19: End(Item)
0..19: End(List(false))
EOF
```

Edit:

Did some more testing and it's possible that more than one list could get parsed. For example:

```markdown
- [x] 3. - * 1) some text
```


```
0..26: Start(List(None))
0..26: Start(Item)
5..26: Start(List(Some(3)))
5..26: Start(Item)
9..26: Start(List(None))
9..26: Start(Item)
11..26: Start(List(None))
11..26: Start(Item)
13..26: Start(List(Some(1)))
13..26: Start(Item)
2..5: TaskListMarker(true)
16..25: Text(Borrowed("some text"))
13..26: End(Item)
13..26: End(List(true))
11..26: End(Item)
11..26: End(List(false))
9..26: End(Item)
9..26: End(List(false))
5..26: End(Item)
5..26: End(List(true))
0..26: End(Item)
0..26: End(List(false))
EOF
```


Pull request description:
Fix tasklist parsing bugs
Fixes #999


Patch:
diff --git a/pulldown-cmark/specs/regression.txt b/pulldown-cmark/specs/regression.txt
index 5d2dcda..cb95bdf 100644
--- a/pulldown-cmark/specs/regression.txt
+++ b/pulldown-cmark/specs/regression.txt
@@ -2798,3 +2798,29 @@ the trailing space after the &gt; should be stripped
 <blockquote></blockquote></li>
 </ul>
 ````````````````````````````````
+
+https://github.com/pulldown-cmark/pulldown-cmark/issues/999
+
+```````````````````````````````` example_wikilinks
+- [x] * some text
+- [ ] > some text
+- [x]
+  * some text
+- [ ]
+  > some text
+.
+<ul>
+<li><input disabled="" type="checkbox" checked=""/>
+* some text</li>
+<li><input disabled="" type="checkbox"/>
+&gt; some text</li>
+<li><input disabled="" type="checkbox" checked=""/>
+<ul>
+<li>some text</li>
+</ul></li>
+<li><input disabled="" type="checkbox"/>
+<blockquote>
+<p>some text</p>
+</blockquote></li>
+</ul>
+````````````````````````````````
diff --git a/pulldown-cmark/src/firstpass.rs b/pulldown-cmark/src/firstpass.rs
index 5fa1258..9874479 100644
--- a/pulldown-cmark/src/firstpass.rs
+++ b/pulldown-cmark/src/firstpass.rs
@@ -34,7 +34,6 @@ pub(crate) fn run_first_pass(text: &str, options: Options) -> (Tree<Item>, Alloc
         allocs: Allocations::new(),
         options,
         lookup_table,
-        next_paragraph_task: None,
         brace_context_next: 0,
         brace_context_stack: Vec::new(),
     };
@@ -61,9 +60,6 @@ struct FirstPass<'a, 'b> {
     allocs: Allocations<'a>,
     options: Options,
     lookup_table: &'b LookupTable,
-    /// This is `Some(item)` when the next paragraph
-    /// starts with a task list marker.
-    next_paragraph_task: Option<Item>,
     /// Math environment brace nesting.
     brace_context_stack: Vec<u8>,
     brace_context_next: usize,
@@ -156,7 +152,9 @@ impl<'a, 'b> FirstPass<'a, 'b> {
                             self.begin_list_item = Some(task_list_marker.end + n);
                             return task_list_marker.end + n;
                         } else {
-                            self.next_paragraph_task = Some(task_list_marker);
+                            line_start.scan_all_space();
+                            let ix = start_ix + line_start.bytes_scanned();
+                            return self.parse_paragraph(ix, Some(task_list_marker));
                         }
                     }
                 }
@@ -407,7 +405,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
 
         let ix = start_ix + line_start.bytes_scanned();
 
-        self.parse_paragraph(ix)
+        self.parse_paragraph(ix, None)
     }
 
     /// footnote definitions and GFM quote markers can be "interrupted"
@@ -576,7 +574,7 @@ impl<'a, 'b> FirstPass<'a, 'b> {
     }
 
     /// Returns offset of line start after paragraph.
-    fn parse_paragraph(&mut self, start_ix: usize) -> usize {
+    fn parse_paragraph(&mut self, start_ix: usize, tasklist_marker: Option<Item>) -> usize {
         let body = if let Some(ItemBody::DefinitionList(_)) =
             self.tree.peek_up().map(|idx| self.tree[idx].item.body)
         {
@@ -604,9 +602,8 @@ impl<'a, 'b> FirstPass<'a, 'b> {
         });
         self.tree.push();
 
-        if let Some(item) = self.next_paragraph_task {
+        if let Some(item) = tasklist_marker {
             self.tree.append(item);
-            self.next_paragraph_task = None;
         }
 
         let bytes = self.text.as_bytes();
diff --git a/pulldown-cmark/src/scanners.rs b/pulldown-cmark/src/scanners.rs
index 5c2c937..19cee81 100644
--- a/pulldown-cmark/src/scanners.rs
+++ b/pulldown-cmark/src/scanners.rs
@@ -404,7 +404,7 @@ impl<'a> LineStart<'a> {
         if !self
             .bytes
             .get(self.ix)
-            .map(|&b| is_ascii_whitespace_no_nl(b))
+            .map(|&b| is_ascii_whitespace(b))
             .unwrap_or(false)
         {
             *self = save;


Test patch:
diff --git a/pulldown-cmark/tests/suite/regression.rs b/pulldown-cmark/tests/suite/regression.rs
index 54fe14a..b3be073 100644
--- a/pulldown-cmark/tests/suite/regression.rs
+++ b/pulldown-cmark/tests/suite/regression.rs
@@ -3339,3 +3339,31 @@ fn regression_test_209() {
 
     test_markdown_html(original, expected, false, false, false, false, true);
 }
+
+#[test]
+fn regression_test_210() {
+    let original = r##"- [x] * some text
+- [ ] > some text
+- [x]
+  * some text
+- [ ]
+  > some text
+"##;
+    let expected = r##"<ul>
+<li><input disabled="" type="checkbox" checked=""/>
+* some text</li>
+<li><input disabled="" type="checkbox"/>
+&gt; some text</li>
+<li><input disabled="" type="checkbox" checked=""/>
+<ul>
+<li>some text</li>
+</ul></li>
+<li><input disabled="" type="checkbox"/>
+<blockquote>
+<p>some text</p>
+</blockquote></li>
+</ul>
+"##;
+
+    test_markdown_html(original, expected, false, false, false, false, true);
+}