Repository: mideind/Tokenizer (language: python)

Issue:
Character omitted
When using split_into_sentences() on the string

@@[smiley: Too Funny: [4/4_1_72]] 

the result is 

@ @ [ smiley : Too Funny : [ 4/4 _ 1 _ 72

so "]]" from the end of the sentence is omitted.




Discussion / hints:

Fix patch:
diff --git a/src/tokenizer/tokenizer.py b/src/tokenizer/tokenizer.py
index 0bff6a6..d19f708 100644
--- a/src/tokenizer/tokenizer.py
+++ b/src/tokenizer/tokenizer.py
@@ -453,6 +453,7 @@ class TOK:
     X_END = 12001
 
     END = frozenset((P_END, S_END, X_END, S_SPLIT))
+    BEGIN = frozenset((P_BEGIN, S_BEGIN))
     TEXT = frozenset((WORD, PERSON, ENTITY, MOLECULE, COMPANY))
     TEXT_EXCL_PERSON = frozenset((WORD, ENTITY, MOLECULE, COMPANY))
 
@@ -1406,7 +1407,7 @@ def parse_tokens(txt: Union[str, Iterable[str]], **options: Any) -> Iterator[Tok
     #    exhausted. At that point, we obtain the next token and start from 2).
 
     rt: Tok
-
+    inside_paragraph_marker: bool = False
     for rt in generate_rough_tokens(
         txt, replace_composite_glyphs, replace_html_escapes, one_sent_per_line
     ):
@@ -1539,13 +1540,18 @@ def parse_tokens(txt: Union[str, Iterable[str]], **options: Any) -> Iterator[Tok
                     # Probably an idiot trying to type opening double quotes with commas
                     punct, rt = rt.split(2)
                     yield TOK.Punctuation(punct, normalized="„")
-                elif lw >= 2 and (rtxt.startswith("[[") or rtxt.startswith("]]")):
+                elif lw >= 2 and (
+                    (rtxt.startswith("[[") and not inside_paragraph_marker)
+                    or (rtxt.startswith("]]") and inside_paragraph_marker)
+                ):
                     # Begin or end paragraph marker
                     marker, rt = rt.split(2)
                     if marker.txt == "[[":
                         yield TOK.Begin_Paragraph(marker)
-                    else:
+                        inside_paragraph_marker = True
+                    elif marker.txt == "]]":
                         yield TOK.End_Paragraph(marker)
+                        inside_paragraph_marker = False
                 elif rtxt[0] in HYPHENS:
                     # Normalize all hyphens the same way
                     punct, rt = rt.split(1)
@@ -2876,7 +2882,7 @@ def split_into_sentences(
                 else:
                     yield " ".join(curr_sent)
             curr_sent = []
-        else:
+        elif t.kind not in TOK.BEGIN:
             txt = to_text(t)
             if txt:
                 curr_sent.append(txt)


Test patch:
diff --git a/test/test_tokenizer.py b/test/test_tokenizer.py
index 8723122..a86f8d5 100755
--- a/test/test_tokenizer.py
+++ b/test/test_tokenizer.py
@@ -1844,6 +1844,28 @@ def gen(s: Iterable[str]) -> Iterator[str]:
     sents = list(g)
     assert len(sents) == 0
 
+    g = t.split_into_sentences("Athugum [hvort [setningin sé rétt skilin]].")
+    sents = list(g)
+    assert len(sents) == 1
+    assert sents == [
+        "Athugum [ hvort [ setningin sé rétt skilin ] ] ."
+    ]
+
+    g = t.split_into_sentences("Þessi [ætti [líka að]] vera rétt skilin.")
+    sents = list(g)
+    assert len(sents) == 1
+    assert sents == [
+        "Þessi [ ætti [ líka að ] ] vera rétt skilin ."
+    ]
+
+    # g = t.split_into_sentences("Þessi á [[líka að]] vera rétt skilin.")
+    # sents = list(g)
+    # assert len(sents) == 3
+    # assert sents == [
+    #     "Þessi á",
+    #     "[[ líka að ]]",
+    #     "vera rétt skilin ."
+    # ]
     # Test onesentperline