Repository: fnl/syntok

Issue:
Under-splitting on "set." and "ago."
Syntok did not split:
> Some of the cookies are essential for parts of the site to operate and have already been set. You may delete and block all cookies from this site, but if you do, parts of the site may not work.

and:

> The sharpshooter appears to be checked out on his Kings experience, and an argument could easily be raised that he should have been moved two years ago. Now, his $23 million salary will be a tough pill for teams to swallow, even if there is decent chance of a solid bounce-back year at a new destination.

Because those are the official abbreviations for two Spanish months (septiembre and agosto).

Comments:

Patch:
diff --git a/syntok/_segmentation_states.py b/syntok/_segmentation_states.py
index b94668b..bc01c02 100644
--- a/syntok/_segmentation_states.py
+++ b/syntok/_segmentation_states.py
@@ -76,7 +76,6 @@ class State(metaclass=ABCMeta):
     univ Univ Urt vda Vda vol Vol vs vta zB zit zzgl
     Mon lun Tue mar Wed mie mié Thu jue Fri vie Sat sab Sun dom
     """.split()
-        + list(months)
     )
     """Abbreviations with no dots inside."""
 
@@ -188,14 +187,13 @@ class State(metaclass=ABCMeta):
         return not self.is_empty and self.__queue[0].value.isnumeric()
 
     @property
-    def next_is_alphanumeric(self) -> bool:
+    def next_is_alphanumeric_containing_numeric_char(self) -> bool:
         if self.is_empty:
             return False
 
         v = self.__queue[0].value
         return (
             any(c.isnumeric() for c in v)
-            and any(c.isalpha() for c in v)
             and v.isalnum()
         )
 
@@ -389,7 +387,7 @@ class State(metaclass=ABCMeta):
         ):
             return self
 
-        elif token_before == "no" and self.next_is_alphanumeric:
+        elif token_before in ("no", "No", "NO") and self.next_is_alphanumeric_containing_numeric_char:
             return self
 
         elif self.next_is_numeric and self.next_has_no_spacing:
@@ -408,6 +406,9 @@ class State(metaclass=ABCMeta):
         elif token_before.isnumeric() and self.next_is_month_abbreviation:
             return self
 
+        elif token_before in State.months and self.next_is_numeric:
+            return self
+
         elif "." in token_before and token_after != ".":
             return self
 


Test patch:
diff --git a/syntok/segmenter_test.py b/syntok/segmenter_test.py
index 48e59c3..60aa0eb 100644
--- a/syntok/segmenter_test.py
+++ b/syntok/segmenter_test.py
@@ -87,6 +87,7 @@ This is a sentence terminal ellipsis...
 This is another sentence terminal ellipsis....
 An easy to handle G. species mention.
 Am 13. Jän. 2006 war es regnerisch.
+And on Jan. 22, 2022 it was, too.
 (Phil. 4:8)
 (Oh. Again!)
 Syntok even handles bible quotes!
@@ -144,6 +145,7 @@ In line with the literature on DLB.
 This is verse 14;45 in the test;
 Splitting on semi-colons.
 The discovery of low-mass nulceli (AGN; NGC 4395 and POX 52; Filippenko & Sargent 1989; Kunth et al. 1987) triggered a quest; it has yielded today more than 500 sources.
+The Company is the No. 2 and No. 3 largest chain in the U.S. and Canada, respectively, by number of stores.
 Always last, clear closing example."""
 
 SENTENCES = OSPL.split("\n")
@@ -276,6 +278,22 @@ class TestSegmenter(TestCase):
         result = segmenter.split(iter(tokens))
         self.assertEqual([tokens[:sep], tokens[sep:]], result)
 
+    def test_sentences_ending_with_false_positive_month_abbreviation_1(self):
+        tokens = Tokenizer().split(
+            "Some of the cookies are essential for parts of the site to operate and have already been set. You may delete and block all cookies from this site, but if you do, parts of the site may not work."
+        )
+        sep = 19
+        result = segmenter.split(iter(tokens))
+        self.assertEqual([tokens[:sep], tokens[sep:]], result)
+
+    def test_sentences_ending_with_false_positive_month_abbreviation_2(self):
+        tokens = Tokenizer().split(
+            "The sharpshooter appears to be checked out on his Kings experience, and an argument could easily be raised that he should have been moved two years ago. Now, his $23 million salary will be a tough pill for teams to swallow, even if there is decent chance of a solid bounce-back year at a new destination."
+        )
+        sep = 29
+        result = segmenter.split(iter(tokens))
+        self.assertEqual([tokens[:sep], tokens[sep:]], result)
+
     def test_sentences_with_enumerations(self):
         tokens = Tokenizer().split("1. This goes first. 2. And here thereafter.")
         sep = 6
@@ -378,9 +396,7 @@ class TestSegmenter(TestCase):
         self.assertEqual([tokens], result)
 
     def test_do_not_split_bible_citation(self):
-        tokens = Tokenizer().split(
-            "This is a bible quote? (Phil. 4:8) Yes, it is!"
-        )
+        tokens = Tokenizer().split("This is a bible quote? (Phil. 4:8) Yes, it is!")
         result = segmenter.split(iter(tokens))
         self.assertEqual(len(result[0]), 6)
         self.assertEqual(len(result[1]), 5)
@@ -463,12 +479,16 @@ class TestSegmenter(TestCase):
         self.assertEqual([tokens], result)
 
     def test_no_split_on_strange_text(self):
-        tokens = Tokenizer().split("Four patients (67%) with an average response of 3.3 mos. (range 6 wks. to 12 mos.)")
+        tokens = Tokenizer().split(
+            "Four patients (67%) with an average response of 3.3 mos. (range 6 wks. to 12 mos.)"
+        )
         result = segmenter.split(iter(tokens))
         self.assertEqual([tokens], result)
 
     def test_no_split_on_strange_text2(self):
-        tokens = Tokenizer().split("Packed cells (PRBC) for less than 20,000 thousand/micro.L, repsectively.")
+        tokens = Tokenizer().split(
+            "Packed cells (PRBC) for less than 20,000 thousand/micro.L, repsectively."
+        )
         result = segmenter.split(iter(tokens))
         self.assertEqual([tokens], result)