Repository: nltk/nltk

Issue:
PunktSentenceTokenizer doesn't work with custom end sentence character
Example:

```
import nltk
from nltk.tokenize.punkt import PunktLanguageVars

class ExtLangVars(PunktLanguageVars):
    sent_end_chars = ('.', '?', '!', '^')

punkt_tk = nltk.data.load('tokenizers/punkt/english.pickle')
punkt_tk._lang_vars = ExtLangVars()

txt1 = 'Subject: Some subject. Attachments: Some attachemnts'
txt2 = 'Subject: Some subject^ Attachments: Some attachemnts'

r = punkt_tk.tokenize(txt1)
print(r)

r = punkt_tk.tokenize(txt2)
print(r)

d = punkt_tk.debug_decisions(txt1)
for x in d:
    print(nltk.tokenize.punkt.format_debug_decision(x))

#d = punkt_tk.debug_decisions(txt2)
#for x in d:
#    print(nltk.tokenize.punkt.format_debug_decision(x))

```
Also if you uncomment last lines, it will fail with index out of bounds. I think it's because of hardcoded 

`        self.period_final = tok.endswith('.')`

in PunktToken. Tried many ways to work around this, but no luck. Any suggestions would be appreciated.


Discussion:

Resolving patch:
diff --git a/nltk/tokenize/punkt.py b/nltk/tokenize/punkt.py
--- a/nltk/tokenize/punkt.py
+++ b/nltk/tokenize/punkt.py
@@ -1285,27 +1285,27 @@ def debug_decisions(self, text):
             decision_text = match.group() + match.group("after_tok")
             tokens = self._tokenize_words(decision_text)
             tokens = list(self._annotate_first_pass(tokens))
-            while not tokens[0].period_final:
+            while tokens and not tokens[0].tok.endswith(self._lang_vars.sent_end_chars):
                 tokens.pop(0)
-            yield dict(
-                period_index=match.end() - 1,
-                text=decision_text,
-                type1=tokens[0].type,
-                type2=tokens[1].type,
-                type1_in_abbrs=bool(tokens[0].abbr),
-                type1_is_initial=bool(tokens[0].is_initial),
-                type2_is_sent_starter=tokens[1].type_no_sentperiod
-                in self._params.sent_starters,
-                type2_ortho_heuristic=self._ortho_heuristic(tokens[1]),
-                type2_ortho_contexts=set(
+            yield {
+                "period_index": match.end() - 1,
+                "text": decision_text,
+                "type1": tokens[0].type,
+                "type2": tokens[1].type,
+                "type1_in_abbrs": bool(tokens[0].abbr),
+                "type1_is_initial": bool(tokens[0].is_initial),
+                "type2_is_sent_starter": tokens[1].type_no_sentperiod 
+                                         in self._params.sent_starters,
+                "type2_ortho_heuristic": self._ortho_heuristic(tokens[1]),
+                "type2_ortho_contexts": set(
                     self._params._debug_ortho_context(tokens[1].type_no_sentperiod)
                 ),
-                collocation=(tokens[0].type_no_sentperiod, tokens[1].type_no_sentperiod)
-                in self._params.collocations,
-                reason=self._second_pass_annotation(tokens[0], tokens[1])
-                or REASON_DEFAULT_DECISION,
-                break_decision=tokens[0].sentbreak,
-            )
+                "collocation": (tokens[0].type_no_sentperiod, tokens[1].type_no_sentperiod)
+                               in self._params.collocations,
+                "reason": self._second_pass_annotation(tokens[0], tokens[1])
+                          or REASON_DEFAULT_DECISION,
+                "break_decision": tokens[0].sentbreak,
+            }
 
     def span_tokenize(self, text, realign_boundaries=True):
         """
@@ -1633,21 +1633,21 @@ def _ortho_heuristic(self, aug_tok):
         return "unknown"
 
 
-DEBUG_DECISION_FMT = """Text: %(text)r (at offset %(period_index)d)
-Sentence break? %(break_decision)s (%(reason)s)
-Collocation? %(collocation)s
-%(type1)r:
-    known abbreviation: %(type1_in_abbrs)s
-    is initial: %(type1_is_initial)s
-%(type2)r:
-    known sentence starter: %(type2_is_sent_starter)s
-    orthographic heuristic suggests is a sentence starter? %(type2_ortho_heuristic)s
-    orthographic contexts in training: %(type2_ortho_contexts)s
+DEBUG_DECISION_FMT = """Text: {text!r} (at offset {period_index})
+Sentence break? {break_decision} ({reason})
+Collocation? {collocation}
+{type1!r}:
+    known abbreviation: {type1_in_abbrs}
+    is initial: {type1_is_initial}
+{type2!r}:
+    known sentence starter: {type2_is_sent_starter}
+    orthographic heuristic suggests is a sentence starter? {type2_ortho_heuristic}
+    orthographic contexts in training: {type2_ortho_contexts}
 """
 
 
 def format_debug_decision(d):
-    return DEBUG_DECISION_FMT % d
+    return DEBUG_DECISION_FMT.format(**d)
 
 
 def demo(text, tok_cls=PunktSentenceTokenizer, train_cls=PunktTrainer):


Test patch:
diff --git a/nltk/test/unit/test_tokenize.py b/nltk/test/unit/test_tokenize.py
--- a/nltk/test/unit/test_tokenize.py
+++ b/nltk/test/unit/test_tokenize.py
@@ -14,24 +14,21 @@
     LegalitySyllableTokenizer,
 )
 
-def setup_module(module):
-    import pytest
 
+def load_stanford_segmenter():
     try:
         seg = StanfordSegmenter()
         seg.default_config("ar")
         seg.default_config("zh")
-    except LookupError as e:
-        pytest.skip(
-            "Tests for nltk.tokenize.stanford_segmenter skipped: %s" % str(e)
-        )
-
-    try:
-        StanfordTokenizer()
+        return True
     except LookupError:
-        pytest.skip(
-            "Tests for nltk.tokenize.stanford are skipped because the stanford postagger jar doesn't exist"
-        )
+        return False
+
+
+check_stanford_segmenter = pytest.mark.skipif(
+    not load_stanford_segmenter(),
+    reason="NLTK was unable to find stanford-segmenter.jar.",
+)
 
 
 class TestTokenize:
@@ -44,16 +41,16 @@ def test_tweet_tokenizer(self):
         s9 = "@myke: Let's test these words: resumé España München français"
         tokens = tokenizer.tokenize(s9)
         expected = [
-            ':',
+            ":",
             "Let's",
-            'test',
-            'these',
-            'words',
-            ':',
-            'resumé',
-            'España',
-            'München',
-            'français',
+            "test",
+            "these",
+            "words",
+            ":",
+            "resumé",
+            "España",
+            "München",
+            "français",
         ]
         assert tokens == expected
 
@@ -62,57 +59,54 @@ def test_sonority_sequencing_syllable_tokenizer(self):
         Test SyllableTokenizer tokenizer.
         """
         tokenizer = SyllableTokenizer()
-        tokens = tokenizer.tokenize('justification')
-        assert tokens == ['jus', 'ti', 'fi', 'ca', 'tion']
+        tokens = tokenizer.tokenize("justification")
+        assert tokens == ["jus", "ti", "fi", "ca", "tion"]
 
     def test_legality_principle_syllable_tokenizer(self):
         """
         Test LegalitySyllableTokenizer tokenizer.
         """
         from nltk.corpus import words
+
         test_word = "wonderful"
         tokenizer = LegalitySyllableTokenizer(words.words())
         tokens = tokenizer.tokenize(test_word)
-        assert tokens == ['won', 'der', 'ful']
+        assert tokens == ["won", "der", "ful"]
 
+    @check_stanford_segmenter
     def test_stanford_segmenter_arabic(self):
         """
         Test the Stanford Word Segmenter for Arabic (default config)
         """
-        try:
-            seg = StanfordSegmenter()
-            seg.default_config('ar')
-            sent = u'يبحث علم الحاسوب استخدام الحوسبة بجميع اشكالها لحل المشكلات'
-            segmented_sent = seg.segment(sent.split())
-            assert segmented_sent.split() == [
-                'يبحث',
-                'علم',
-                'الحاسوب',
-                'استخدام',
-                'الحوسبة',
-                'ب',
-                'جميع',
-                'اشكال',
-                'ها',
-                'ل',
-                'حل',
-                'المشكلات',
-            ]
-        except LookupError as e:
-            pytest.skip(str(e))
+        seg = StanfordSegmenter()
+        seg.default_config("ar")
+        sent = u"يبحث علم الحاسوب استخدام الحوسبة بجميع اشكالها لحل المشكلات"
+        segmented_sent = seg.segment(sent.split())
+        assert segmented_sent.split() == [
+            "يبحث",
+            "علم",
+            "الحاسوب",
+            "استخدام",
+            "الحوسبة",
+            "ب",
+            "جميع",
+            "اشكال",
+            "ها",
+            "ل",
+            "حل",
+            "المشكلات",
+        ]
 
+    @check_stanford_segmenter
     def test_stanford_segmenter_chinese(self):
         """
         Test the Stanford Word Segmenter for Chinese (default config)
         """
-        try:
-            seg = StanfordSegmenter()
-            seg.default_config('zh')
-            sent = u"这是斯坦福中文分词器测试"
-            segmented_sent = seg.segment(sent.split())
-            assert segmented_sent.split() == ['这', '是', '斯坦福', '中文', '分词器', '测试']
-        except LookupError as e:
-            pytest.skip(str(e))
+        seg = StanfordSegmenter()
+        seg.default_config("zh")
+        sent = u"这是斯坦福中文分词器测试"
+        segmented_sent = seg.segment(sent.split())
+        assert segmented_sent.split() == ["这", "是", "斯坦福", "中文", "分词器", "测试"]
 
     def test_phone_tokenizer(self):
         """
@@ -122,14 +116,14 @@ def test_phone_tokenizer(self):
         # Should be recognized as a phone number, albeit one with multiple spaces
         tokenizer = TweetTokenizer()
         test1 = "(393)  928 -3010"
-        expected = ['(393)  928 -3010']
+        expected = ["(393)  928 -3010"]
         result = tokenizer.tokenize(test1)
         assert result == expected
 
         # Due to newline, first three elements aren't part of a phone number;
         # fourth is
         test2 = "(393)\n928 -3010"
-        expected = ['(', '393', ')', "928 -3010"]
+        expected = ["(", "393", ")", "928 -3010"]
         result = tokenizer.tokenize(test2)
         assert result == expected
 
@@ -138,8 +132,21 @@ def test_pad_asterisk(self):
         Test padding of asterisk for word tokenization.
         """
         text = "This is a, *weird sentence with *asterisks in it."
-        expected = ['This', 'is', 'a', ',', '*', 'weird', 'sentence',
-                    'with', '*', 'asterisks', 'in', 'it', '.']
+        expected = [
+            "This",
+            "is",
+            "a",
+            ",",
+            "*",
+            "weird",
+            "sentence",
+            "with",
+            "*",
+            "asterisks",
+            "in",
+            "it",
+            ".",
+        ]
         assert word_tokenize(text) == expected
 
     def test_pad_dotdot(self):
@@ -147,9 +154,24 @@ def test_pad_dotdot(self):
         Test padding of dotdot* for word tokenization.
         """
         text = "Why did dotdot.. not get tokenized but dotdotdot... did? How about manydots....."
-        expected = ['Why', 'did', 'dotdot', '..', 'not', 'get',
-                    'tokenized', 'but', 'dotdotdot', '...', 'did', '?',
-                    'How', 'about', 'manydots', '.....']
+        expected = [
+            "Why",
+            "did",
+            "dotdot",
+            "..",
+            "not",
+            "get",
+            "tokenized",
+            "but",
+            "dotdotdot",
+            "...",
+            "did",
+            "?",
+            "How",
+            "about",
+            "manydots",
+            ".....",
+        ]
         assert word_tokenize(text) == expected
 
     def test_remove_handle(self):
@@ -161,42 +183,42 @@ def test_remove_handle(self):
 
         # Simple example. Handles with just numbers should be allowed
         test1 = "@twitter hello @twi_tter_. hi @12345 @123news"
-        expected = ['hello', '.', 'hi']
+        expected = ["hello", ".", "hi"]
         result = tokenizer.tokenize(test1)
         assert result == expected
 
         # Handles are allowed to follow any of the following characters
         test2 = "@n`@n~@n(@n)@n-@n=@n+@n\\@n|@n[@n]@n{@n}@n;@n:@n'@n\"@n/@n?@n.@n,@n<@n>@n @n\n@n ñ@n.ü@n.ç@n."
         expected = [
-            '`',
-            '~',
-            '(',
-            ')',
-            '-',
-            '=',
-            '+',
-            '\\',
-            '|',
-            '[',
-            ']',
-            '{',
-            '}',
-            ';',
-            ':',
+            "`",
+            "~",
+            "(",
+            ")",
+            "-",
+            "=",
+            "+",
+            "\\",
+            "|",
+            "[",
+            "]",
+            "{",
+            "}",
+            ";",
+            ":",
             "'",
             '"',
-            '/',
-            '?',
-            '.',
-            ',',
-            '<',
-            '>',
-            'ñ',
-            '.',
-            'ü',
-            '.',
-            'ç',
-            '.',
+            "/",
+            "?",
+            ".",
+            ",",
+            "<",
+            ">",
+            "ñ",
+            ".",
+            "ü",
+            ".",
+            "ç",
+            ".",
         ]
         result = tokenizer.tokenize(test2)
         assert result == expected
@@ -204,103 +226,103 @@ def test_remove_handle(self):
         # Handles are NOT allowed to follow any of the following characters
         test3 = "a@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@n"
         expected = [
-            'a',
-            '@n',
-            'j',
-            '@n',
-            'z',
-            '@n',
-            'A',
-            '@n',
-            'L',
-            '@n',
-            'Z',
-            '@n',
-            '1',
-            '@n',
-            '4',
-            '@n',
-            '7',
-            '@n',
-            '9',
-            '@n',
-            '0',
-            '@n',
-            '_',
-            '@n',
-            '!',
-            '@n',
-            '@',
-            '@n',
-            '#',
-            '@n',
-            '$',
-            '@n',
-            '%',
-            '@n',
-            '&',
-            '@n',
-            '*',
-            '@n',
+            "a",
+            "@n",
+            "j",
+            "@n",
+            "z",
+            "@n",
+            "A",
+            "@n",
+            "L",
+            "@n",
+            "Z",
+            "@n",
+            "1",
+            "@n",
+            "4",
+            "@n",
+            "7",
+            "@n",
+            "9",
+            "@n",
+            "0",
+            "@n",
+            "_",
+            "@n",
+            "!",
+            "@n",
+            "@",
+            "@n",
+            "#",
+            "@n",
+            "$",
+            "@n",
+            "%",
+            "@n",
+            "&",
+            "@n",
+            "*",
+            "@n",
         ]
         result = tokenizer.tokenize(test3)
         assert result == expected
 
         # Handles are allowed to precede the following characters
         test4 = "@n!a @n#a @n$a @n%a @n&a @n*a"
-        expected = ['!', 'a', '#', 'a', '$', 'a', '%', 'a', '&', 'a', '*', 'a']
+        expected = ["!", "a", "#", "a", "$", "a", "%", "a", "&", "a", "*", "a"]
         result = tokenizer.tokenize(test4)
         assert result == expected
 
         # Tests interactions with special symbols and multiple @
         test5 = "@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@n"
         expected = [
-            '!',
-            '@n',
-            '#',
-            '@n',
-            '$',
-            '@n',
-            '%',
-            '@n',
-            '&',
-            '@n',
-            '*',
-            '@n',
-            '@n',
-            '@n',
-            '@',
-            '@n',
-            '@n',
-            '@',
-            '@n',
-            '@n_',
-            '@n',
-            '@n7',
-            '@n',
-            '@nj',
-            '@n',
+            "!",
+            "@n",
+            "#",
+            "@n",
+            "$",
+            "@n",
+            "%",
+            "@n",
+            "&",
+            "@n",
+            "*",
+            "@n",
+            "@n",
+            "@n",
+            "@",
+            "@n",
+            "@n",
+            "@",
+            "@n",
+            "@n_",
+            "@n",
+            "@n7",
+            "@n",
+            "@nj",
+            "@n",
         ]
         result = tokenizer.tokenize(test5)
         assert result == expected
 
         # Tests that handles can have a max length of 20
         test6 = "@abcdefghijklmnopqrstuvwxyz @abcdefghijklmnopqrst1234 @abcdefghijklmnopqrst_ @abcdefghijklmnopqrstendofhandle"
-        expected = ['uvwxyz', '1234', '_', 'endofhandle']
+        expected = ["uvwxyz", "1234", "_", "endofhandle"]
         result = tokenizer.tokenize(test6)
         assert result == expected
 
         # Edge case where an @ comes directly after a long handle
         test7 = "@abcdefghijklmnopqrstu@abcde @abcdefghijklmnopqrst@abcde @abcdefghijklmnopqrst_@abcde @abcdefghijklmnopqrst5@abcde"
         expected = [
-            'u',
-            '@abcde',
-            '@abcdefghijklmnopqrst',
-            '@abcde',
-            '_',
-            '@abcde',
-            '5',
-            '@abcde',
+            "u",
+            "@abcde",
+            "@abcdefghijklmnopqrst",
+            "@abcde",
+            "_",
+            "@abcde",
+            "5",
+            "@abcde",
         ]
         result = tokenizer.tokenize(test7)
         assert result == expected
@@ -343,7 +365,7 @@ def test_treebank_span_tokenizer(self):
         assert result == expected
 
         # Test case with double quotation
-        test2 = "The DUP is similar to the \"religious right\" in the United States and takes a hardline stance on social issues"
+        test2 = 'The DUP is similar to the "religious right" in the United States and takes a hardline stance on social issues'
         expected = [
             (0, 3),
             (4, 7),
@@ -408,20 +430,35 @@ def test_word_tokenize(self):
         """
 
         sentence = "The 'v', I've been fooled but I'll seek revenge."
-        expected = ['The', "'", 'v', "'", ',', 'I', "'ve", 'been', 'fooled',
-                    'but', 'I', "'ll", 'seek', 'revenge', '.']
+        expected = [
+            "The",
+            "'",
+            "v",
+            "'",
+            ",",
+            "I",
+            "'ve",
+            "been",
+            "fooled",
+            "but",
+            "I",
+            "'ll",
+            "seek",
+            "revenge",
+            ".",
+        ]
         assert word_tokenize(sentence) == expected
 
         sentence = "'v' 're'"
-        expected = ["'", 'v', "'", "'re", "'"]
+        expected = ["'", "v", "'", "'re", "'"]
         assert word_tokenize(sentence) == expected
 
     def test_punkt_pair_iter(self):
 
         test_cases = [
-            ('12', [('1', '2'), ('2', None)]),
-            ('123', [('1', '2'), ('2', '3'), ('3', None)]),
-            ('1234', [('1', '2'), ('2', '3'), ('3', '4'), ('4', None)]),
+            ("12", [("1", "2"), ("2", None)]),
+            ("123", [("1", "2"), ("2", "3"), ("3", None)]),
+            ("1234", [("1", "2"), ("2", "3"), ("3", "4"), ("4", None)]),
         ]
 
         for (test_input, expected_output) in test_cases:
@@ -446,18 +483,23 @@ def word_tokenize(self, s):
 
         obj._lang_vars = TestPunktTokenizeWordsMock()
         # unpack generator, ensure that no error is raised
-        list(obj._tokenize_words('test'))
+        list(obj._tokenize_words("test"))
 
     def test_punkt_tokenize_custom_lang_vars(self):
 
         # Create LangVars including a full stop end character as used in Bengali
         class BengaliLanguageVars(punkt.PunktLanguageVars):
-            sent_end_chars = ('.', '?', '!', '\u0964')
-        obj = punkt.PunktSentenceTokenizer(lang_vars = BengaliLanguageVars())
+            sent_end_chars = (".", "?", "!", "\u0964")
+
+        obj = punkt.PunktSentenceTokenizer(lang_vars=BengaliLanguageVars())
 
         # We now expect these sentences to be split up into the individual sentences
         sentences = u"উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
-        expected = ["উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন।", "অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন।", "এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"]
+        expected = [
+            "উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন।",
+            "অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন।",
+            "এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।",
+        ]
 
         assert obj.tokenize(sentences) == expected
 
@@ -467,6 +509,48 @@ def test_punkt_tokenize_no_custom_lang_vars(self):
 
         # We expect these sentences to not be split properly, as the Bengali full stop '।' is not included in the default language vars
         sentences = u"উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
-        expected = ["উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"]
+        expected = [
+            "উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
+        ]
 
         assert obj.tokenize(sentences) == expected
+
+    @pytest.mark.parametrize(
+        "input_text,n_sents,n_splits,lang_vars",
+        [
+            # Test debug_decisions on a text with two sentences, split by a dot.
+            ("Subject: Some subject. Attachments: Some attachments", 2, 1),
+            # The sentence should be split into two sections,
+            # with one split and hence one decision.
+            # Test debug_decisions on a text with two sentences, split by an exclamation mark.
+            ("Subject: Some subject! Attachments: Some attachments", 2, 1),
+            # The sentence should be split into two sections,
+            # with one split and hence one decision.
+            # Test debug_decisions on a text with one sentences,
+            # which is not split.
+            ("This is just a normal sentence, just like any other.", 1, 0)
+            # Hence just 1
+        ],
+    )
+    def punkt_debug_decisions(self, input_text, n_sents, n_splits, lang_vars=None):
+        tokenizer = punkt.PunktSentenceTokenizer()
+        if lang_vars != None:
+            tokenizer._lang_vars = lang_vars
+
+        assert len(tokenizer.tokenize(input_text)) == n_sents
+        assert len(list(tokenizer.debug_decisions(input_text))) == n_splits
+
+    def test_punkt_debug_decisions_custom_end(self):
+        # Test debug_decisions on a text with two sentences,
+        # split by a custom end character, based on Issue #2519
+        class ExtLangVars(punkt.PunktLanguageVars):
+            sent_end_chars = (".", "?", "!", "^")
+
+        self.punkt_debug_decisions(
+            "Subject: Some subject^ Attachments: Some attachments",
+            n_sents=2,
+            n_splits=1,
+            lang_vars=ExtLangVars(),
+        )
+        # The sentence should be split into two sections,
+        # with one split and hence one decision.