Repository: frostming/marko

Issue:
Crash when parsing "- {NBSP}A"
Code that crashes is:
~~~
parser = Markdown(Parser, MarkdownRenderer)
with open(src_file_path, "r") as txtFile:
   result = parser.parse(txtFile.read())
~~~

Markdown file is:
~~~
-    A
~~~
Note that there is a unicode character U+00A0 right before the `A` that is a NBSP (non-breaking space).

Crash is:
~~~
File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/__init__.py", line 120, in parse
    return cast("Document", self.parser.parse(text))
  File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/parser.py", line 65, in parse
    return self.block_elements["Document"](source_or_text)  # type: ignore
  File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/block.py", line 91, in __init__
    self.children = parser.parse(source)  # type: ignore
  File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/parser.py", line 72, in parse
    result = ele_type.parse(source_or_text)
  File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/block.py", line 474, in parse
    el = parser.block_elements["ListItem"].parse(source)  # type: ignore
  File "/Users/ericzinda/Enlistments/docsproto/env/lib/python3.7/site-packages/marko/block.py", line 562, in parse
    if not source.next_line().strip():  # type: ignore
AttributeError: 'NoneType' object has no attribute 'strip'
~~~


Comments:

Patch:
diff --git a/marko/block.py b/marko/block.py
index 22cf1c5..10a2b9d 100644
--- a/marko/block.py
+++ b/marko/block.py
@@ -8,7 +8,7 @@ from typing import Match, Optional, Tuple, Union, cast
 
 from . import inline, patterns
 from .element import Element
-from .helpers import Source, is_paired, normalize_label
+from .helpers import Source, is_paired, normalize_label, partition_by_spaces
 from .parser import Parser
 
 __all__ = (
@@ -231,7 +231,7 @@ class FencedCode(BlockElement):
         prefix, leading, info = m.groups()
         if leading[0] == "`" and "`" in info:
             return None
-        lang, extra = (info.split(None, 1) + [""] * 2)[:2]
+        lang, _, extra = partition_by_spaces(info)
         cls._parse_info = prefix, leading, lang, extra
         return m
 
@@ -374,7 +374,7 @@ class Paragraph(BlockElement):
             return True
         if parser.block_elements["List"].match(source):
             result = parser.block_elements["ListItem"].parse_leading(
-                source.next_line(), 0
+                source.next_line().rstrip(), 0
             )
             if lazy or (result[1][:-1] == "1" or result[1] in "*-+") and result[3]:
                 return True
@@ -501,13 +501,13 @@ class List(BlockElement):
 class ListItem(BlockElement):
     """List item element. It can only be created by List.parse"""
 
-    _parse_info = (0, "", 0, "")
+    _parse_info = (0, "", 0)
     virtual = True
     _tight = False
     pattern = re.compile(r" {,3}(\d{1,9}[.)]|[*\-+])[ \t\n\r\f]")
 
     def __init__(self) -> None:
-        indent, bullet, mid, tail = self._parse_info
+        indent, bullet, mid = self._parse_info
         self._prefix = " " * indent + re.escape(bullet) + " " * mid
         self._second_prefix = " " * (len(bullet) + indent + (mid or 1))
 
@@ -515,18 +515,10 @@ class ListItem(BlockElement):
     def parse_leading(cls, line: str, prefix_pos: int) -> Tuple[int, str, int, str]:
         stripped_line = line[prefix_pos:].expandtabs(4).lstrip()
         indent = len(line) - prefix_pos - len(stripped_line)
-        temp = stripped_line.split(None, 1)
-        bullet = temp[0]
-        if len(temp) == 1:
-            # Bare bullets (* \n)
-            mid = 0
-            tail = ""
-        else:
-            mid = len(stripped_line) - len("".join(temp))
-            if mid > 4:
-                # If indented more than 4 spaces, it should be a indented code block
-                mid = 1
-            tail = temp[1]
+        bullet, spaces, tail = partition_by_spaces(stripped_line)
+        mid = len(spaces)
+        if mid > 4:
+            mid = 1
         return indent, bullet, mid, tail
 
     @classmethod
@@ -540,7 +532,7 @@ class ListItem(BlockElement):
         m = re.match(source.prefix, next_line)
         if m is not None:
             prefix_pos = m.end()
-        indent, bullet, mid, tail = cls.parse_leading(next_line, prefix_pos)  # type: ignore
+        indent, bullet, mid, _ = cls.parse_leading(next_line.rstrip(), prefix_pos)  # type: ignore
         parent = source.state
         assert isinstance(parent, List)
         if (
@@ -551,7 +543,7 @@ class ListItem(BlockElement):
             return False
         if not parent.ordered and bullet != parent.bullet:
             return False
-        cls._parse_info = (indent, bullet, mid, tail)
+        cls._parse_info = (indent, bullet, mid)
         return True
 
     @classmethod
diff --git a/marko/helpers.py b/marko/helpers.py
index acd09f0..1a27833 100644
--- a/marko/helpers.py
+++ b/marko/helpers.py
@@ -13,6 +13,7 @@ from typing import (
     Match,
     Optional,
     Pattern,
+    Tuple,
     Union,
 )
 
@@ -187,6 +188,27 @@ def normalize_label(label: str) -> str:
     return re.sub(r"\s+", " ", label).strip().casefold()
 
 
+def partition_by_spaces(text: str) -> Tuple[str, str, str]:
+    """Split the given text by spaces or tabs, and return a tuple of
+    (start, delimiter, remaining). If spaces are not found, the latter
+    two elements will be empty.
+    """
+    start = end = -1
+    for i, c in enumerate(text):
+        if c in " \t":
+            if start >= 0:
+                continue
+            start = i
+        elif start >= 0:
+            end = i
+            break
+    if start < 0:
+        return text, "", ""
+    if end < 0:
+        return text[:start], text[start:], ""
+    return text[:start], text[start:end], text[end:]
+
+
 def load_extension_object(name: str) -> Callable:
     """Load extension object from a string.
     First try `marko.ext.<name>` if possible


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 3b472aa..ebefd36 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -57,3 +57,17 @@ def test_load_illegal_extension_object():
         match="Module marko.block does not have 'make_extension' attributte",
     ):
         helpers.load_extension_object("marko.block")()
+
+
+@pytest.mark.parametrize(
+    "text, expected",
+    [
+        ("hello world", ("hello", " ", "world")),
+        ("hello", ("hello", "", "")),
+        ("hello ", ("hello", " ", "")),
+        (" hello", ("", " ", "hello")),
+        ("hello\t  wor ld", ("hello", "\t  ", "wor ld")),
+    ],
+)
+def test_partition_by_spaces(text, expected):
+    assert helpers.partition_by_spaces(text) == expected
diff --git a/tests/test_spec.py b/tests/test_spec.py
index 92c22a9..6a67e28 100644
--- a/tests/test_spec.py
+++ b/tests/test_spec.py
@@ -16,6 +16,11 @@ class TestCommonMark(SpecTestSuite):
         )
         self.assert_case(md, html)
 
+    def test_parse_nbsp_no_crash(self):
+        md = "- \xa0A"
+        html = "<ul>\n<li>A</li>\n</ul>"
+        self.assert_case(md, html)
+
 
 TestCommonMark.load_spec("commonmark")