Repository: nltk/nltk

Issue:
PunktSentenceTokenizer doesn't work with custom end sentence character
Example:

```
import nltk
from nltk.tokenize.punkt import PunktLanguageVars

class ExtLangVars(PunktLanguageVars):
    sent_end_chars = ('.', '?', '!', '^')

punkt_tk = nltk.data.load('tokenizers/punkt/english.pickle')
punkt_tk._lang_vars = ExtLangVars()

txt1 = 'Subject: Some subject. Attachments: Some attachemnts'
txt2 = 'Subject: Some subject^ Attachments: Some attachemnts'

r = punkt_tk.tokenize(txt1)
print(r)

r = punkt_tk.tokenize(txt2)
print(r)

d = punkt_tk.debug_decisions(txt1)
for x in d:
    print(nltk.tokenize.punkt.format_debug_decision(x))

#d = punkt_tk.debug_decisions(txt2)
#for x in d:
#    print(nltk.tokenize.punkt.format_debug_decision(x))

```
Also if you uncomment last lines, it will fail with index out of bounds. I think it's because of hardcoded 

`        self.period_final = tok.endswith('.')`

in PunktToken. Tried many ways to work around this, but no luck. Any suggestions would be appreciated.

Discussion:
purificant: Like how this now runs more tests instead of skipping all. :+1: 
stevenbird: @tomaarsen - I really appreciate the solid documentation and careful testing.

dannysepler: agree with everyone -- great idea, and very thoughtful work
tomaarsen: I've applied your suggestions and quickly ran black over the `test_tokenize` file.

Patch:
diff --git a/nltk/tokenize/punkt.py b/nltk/tokenize/punkt.py
index 512f6630e..9f04be468 100644
--- a/nltk/tokenize/punkt.py
+++ b/nltk/tokenize/punkt.py
@@ -1285,27 +1285,27 @@ class PunktSentenceTokenizer(PunktBaseClass, TokenizerI):
             decision_text = match.group() + match.group("after_tok")
             tokens = self._tokenize_words(decision_text)
             tokens = list(self._annotate_first_pass(tokens))
-            while not tokens[0].period_final:
+            while tokens and not tokens[0].tok.endswith(self._lang_vars.sent_end_chars):
                 tokens.pop(0)
-            yield dict(
-                period_index=match.end() - 1,
-                text=decision_text,
-                type1=tokens[0].type,
-                type2=tokens[1].type,
-                type1_in_abbrs=bool(tokens[0].abbr),
-                type1_is_initial=bool(tokens[0].is_initial),
-                type2_is_sent_starter=tokens[1].type_no_sentperiod
-                in self._params.sent_starters,
-                type2_ortho_heuristic=self._ortho_heuristic(tokens[1]),
-                type2_ortho_contexts=set(
+            yield {
+                "period_index": match.end() - 1,
+                "text": decision_text,
+                "type1": tokens[0].type,
+                "type2": tokens[1].type,
+                "type1_in_abbrs": bool(tokens[0].abbr),
+                "type1_is_initial": bool(tokens[0].is_initial),
+                "type2_is_sent_starter": tokens[1].type_no_sentperiod 
+                                         in self._params.sent_starters,
+                "type2_ortho_heuristic": self._ortho_heuristic(tokens[1]),
+                "type2_ortho_contexts": set(
                     self._params._debug_ortho_context(tokens[1].type_no_sentperiod)
                 ),
-                collocation=(tokens[0].type_no_sentperiod, tokens[1].type_no_sentperiod)
-                in self._params.collocations,
-                reason=self._second_pass_annotation(tokens[0], tokens[1])
-                or REASON_DEFAULT_DECISION,
-                break_decision=tokens[0].sentbreak,
-            )
+                "collocation": (tokens[0].type_no_sentperiod, tokens[1].type_no_sentperiod)
+                               in self._params.collocations,
+                "reason": self._second_pass_annotation(tokens[0], tokens[1])
+                          or REASON_DEFAULT_DECISION,
+                "break_decision": tokens[0].sentbreak,
+            }
 
     def span_tokenize(self, text, realign_boundaries=True):
         """
@@ -1633,21 +1633,21 @@ class PunktSentenceTokenizer(PunktBaseClass, TokenizerI):
         return "unknown"
 
 
-DEBUG_DECISION_FMT = """Text: %(text)r (at offset %(period_index)d)
-Sentence break? %(break_decision)s (%(reason)s)
-Collocation? %(collocation)s
-%(type1)r:
-    known abbreviation: %(type1_in_abbrs)s
-    is initial: %(type1_is_initial)s
-%(type2)r:
-    known sentence starter: %(type2_is_sent_starter)s
-    orthographic heuristic suggests is a sentence starter? %(type2_ortho_heuristic)s
-    orthographic contexts in training: %(type2_ortho_contexts)s
+DEBUG_DECISION_FMT = """Text: {text!r} (at offset {period_index})
+Sentence break? {break_decision} ({reason})
+Collocation? {collocation}
+{type1!r}:
+    known abbreviation: {type1_in_abbrs}
+    is initial: {type1_is_initial}
+{type2!r}:
+    known sentence starter: {type2_is_sent_starter}
+    orthographic heuristic suggests is a sentence starter? {type2_ortho_heuristic}
+    orthographic contexts in training: {type2_ortho_contexts}
 """
 
 
 def format_debug_decision(d):
-    return DEBUG_DECISION_FMT % d
+    return DEBUG_DECISION_FMT.format(**d)
 
 
 def demo(text, tok_cls=PunktSentenceTokenizer, train_cls=PunktTrainer):


Test patch:
diff --git a/nltk/test/unit/test_tokenize.py b/nltk/test/unit/test_tokenize.py
index 79110e3b4..9152ccd09 100644
--- a/nltk/test/unit/test_tokenize.py
+++ b/nltk/test/unit/test_tokenize.py
@@ -14,24 +14,21 @@ from nltk.tokenize import (
     LegalitySyllableTokenizer,
 )
 
-def setup_module(module):
-    import pytest
 
+def load_stanford_segmenter():
     try:
         seg = StanfordSegmenter()
         seg.default_config("ar")
         seg.default_config("zh")
-    except LookupError as e:
-        pytest.skip(
-            "Tests for nltk.tokenize.stanford_segmenter skipped: %s" % str(e)
-        )
-
-    try:
-        StanfordTokenizer()
+        return True
     except LookupError:
-        pytest.skip(
-            "Tests for nltk.tokenize.stanford are skipped because the stanford postagger jar doesn't exist"
-        )
+        return False
+
+
+check_stanford_segmenter = pytest.mark.skipif(
+    not load_stanford_segmenter(),
+    reason="NLTK was unable to find stanford-segmenter.jar.",
+)
 
 
 class TestTokenize:
@@ -44,16 +41,16 @@ class TestTokenize:
         s9 = "@myke: Let's test these words: resumé España München français"
         tokens = tokenizer.tokenize(s9)
         expected = [
-            ':',
+            ":",
             "Let's",
-            'test',
-            'these',
-            'words',
-            ':',
-            'resumé',
-            'España',
-            'München',
-            'français',
+            "test",
+            "these",
+            "words",
+            ":",
+            "resumé",
+            "España",
+            "München",
+            "français",
         ]
         assert tokens == expected
 
@@ -62,57 +59,54 @@ class TestTokenize:
         Test SyllableTokenizer tokenizer.
         """
         tokenizer = SyllableTokenizer()
-        tokens = tokenizer.tokenize('justification')
-        assert tokens == ['jus', 'ti', 'fi', 'ca', 'tion']
+        tokens = tokenizer.tokenize("justification")
+        assert tokens == ["jus", "ti", "fi", "ca", "tion"]
 
     def test_legality_principle_syllable_tokenizer(self):
         """
         Test LegalitySyllableTokenizer tokenizer.
         """
         from nltk.corpus import words
+
         test_word = "wonderful"
         tokenizer = LegalitySyllableTokenizer(words.words())
         tokens = tokenizer.tokenize(test_word)
-        assert tokens == ['won', 'der', 'ful']
+        assert tokens == ["won", "der", "ful"]
 
+    @check_stanford_segmenter
     def test_stanford_segmenter_arabic(self):
         """
         Test the Stanford Word Segmenter for Arabic (default config)
         """
-        try:
-            seg = StanfordSegmenter()
-            seg.default_config('ar')
-            sent = u'يبحث علم الحاسوب استخدام الحوسبة بجميع اشكالها لحل المشكلات'
-            segmented_sent = seg.segment(sent.split())
-            assert segmented_sent.split() == [
-                'يبحث',
-                'علم',
-                'الحاسوب',
-                'استخدام',
-                'الحوسبة',
-                'ب',
-                'جميع',
-                'اشكال',
-                'ها',
-                'ل',
-                'حل',
-                'المشكلات',
-            ]
-        except LookupError as e:
-            pytest.skip(str(e))
+        seg = StanfordSegmenter()
+        seg.default_config("ar")
+        sent = u"يبحث علم الحاسوب استخدام الحوسبة بجميع اشكالها لحل المشكلات"
+        segmented_sent = seg.segment(sent.split())
+        assert segmented_sent.split() == [
+            "يبحث",
+            "علم",
+            "الحاسوب",
+            "استخدام",
+            "الحوسبة",
+            "ب",
+            "جميع",
+            "اشكال",
+            "ها",
+            "ل",
+            "حل",
+            "المشكلات",
+        ]
 
+    @check_stanford_segmenter
     def test_stanford_segmenter_chinese(self):
         """
         Test the Stanford Word Segmenter for Chinese (default config)
         """
-        try:
-            seg = StanfordSegmenter()
-            seg.default_config('zh')
-            sent = u"这是斯坦福中文分词器测试"
-            segmented_sent = seg.segment(sent.split())
-            assert segmented_sent.split() == ['这', '是', '斯坦福', '中文', '分词器', '测试']
-        except LookupError as e:
-            pytest.skip(str(e))
+        seg = StanfordSegmenter()
+        seg.default_config("zh")
+        sent = u"这是斯坦福中文分词器测试"
+        segmented_sent = seg.segment(sent.split())
+        assert segmented_sent.split() == ["这", "是", "斯坦福", "中文", "分词器", "测试"]
 
     def test_phone_tokenizer(self):
         """
@@ -122,14 +116,14 @@ class TestTokenize:
         # Should be recognized as a phone number, albeit one with multiple spaces
         tokenizer = TweetTokenizer()
         test1 = "(393)  928 -3010"
-        expected = ['(393)  928 -3010']
+        expected = ["(393)  928 -3010"]
         result = tokenizer.tokenize(test1)
         assert result == expected
 
         # Due to newline, first three elements aren't part of a phone number;
         # fourth is
         test2 = "(393)\n928 -3010"
-        expected = ['(', '393', ')', "928 -3010"]
+        expected = ["(", "393", ")", "928 -3010"]
         result = tokenizer.tokenize(test2)
         assert result == expected
 
@@ -138,8 +132,21 @@ class TestTokenize:
         Test padding of asterisk for word tokenization.
         """
         text = "This is a, *weird sentence with *asterisks in it."
-        expected = ['This', 'is', 'a', ',', '*', 'weird', 'sentence',
-                    'with', '*', 'asterisks', 'in', 'it', '.']
+        expected = [
+            "This",
+            "is",
+            "a",
+            ",",
+            "*",
+            "weird",
+            "sentence",
+            "with",
+            "*",
+            "asterisks",
+            "in",
+            "it",
+            ".",
+        ]
         assert word_tokenize(text) == expected
 
     def test_pad_dotdot(self):
@@ -147,9 +154,24 @@ class TestTokenize:
         Test padding of dotdot* for word tokenization.
         """
         text = "Why did dotdot.. not get tokenized but dotdotdot... did? How about manydots....."
-        expected = ['Why', 'did', 'dotdot', '..', 'not', 'get',
-                    'tokenized', 'but', 'dotdotdot', '...', 'did', '?',
-                    'How', 'about', 'manydots', '.....']
+        expected = [
+            "Why",
+            "did",
+            "dotdot",
+            "..",
+            "not",
+            "get",
+            "tokenized",
+            "but",
+            "dotdotdot",
+            "...",
+            "did",
+            "?",
+            "How",
+            "about",
+            "manydots",
+            ".....",
+        ]
         assert word_tokenize(text) == expected
 
     def test_remove_handle(self):
@@ -161,42 +183,42 @@ class TestTokenize:
 
         # Simple example. Handles with just numbers should be allowed
         test1 = "@twitter hello @twi_tter_. hi @12345 @123news"
-        expected = ['hello', '.', 'hi']
+        expected = ["hello", ".", "hi"]
         result = tokenizer.tokenize(test1)
         assert result == expected
 
         # Handles are allowed to follow any of the following characters
         test2 = "@n`@n~@n(@n)@n-@n=@n+@n\\@n|@n[@n]@n{@n}@n;@n:@n'@n\"@n/@n?@n.@n,@n<@n>@n @n\n@n ñ@n.ü@n.ç@n."
         expected = [
-            '`',
-            '~',
-            '(',
-            ')',
-            '-',
-            '=',
-            '+',
-            '\\',
-            '|',
-            '[',
-            ']',
-            '{',
-            '}',
-            ';',
-            ':',
+            "`",
+            "~",
+            "(",
+            ")",
+            "-",
+            "=",
+            "+",
+            "\\",
+            "|",
+            "[",
+            "]",
+            "{",
+            "}",
+            ";",
+            ":",
             "'",
             '"',
-            '/',
-            '?',
-            '.',
-            ',',
-            '<',
-            '>',
-            'ñ',
-            '.',
-            'ü',
-            '.',
-            'ç',
-            '.',
+            "/",
+            "?",
+            ".",
+            ",",
+            "<",
+            ">",
+            "ñ",
+            ".",
+            "ü",
+            ".",
+            "ç",
+            ".",
         ]
         result = tokenizer.tokenize(test2)
         assert result == expected
@@ -204,103 +226,103 @@ class TestTokenize:
         # Handles are NOT allowed to follow any of the following characters
         test3 = "a@n j@n z@n A@n L@n Z@n 1@n 4@n 7@n 9@n 0@n _@n !@n @@n #@n $@n %@n &@n *@n"
         expected = [
-            'a',
-            '@n',
-            'j',
-            '@n',
-            'z',
-            '@n',
-            'A',
-            '@n',
-            'L',
-            '@n',
-            'Z',
-            '@n',
-            '1',
-            '@n',
-            '4',
-            '@n',
-            '7',
-            '@n',
-            '9',
-            '@n',
-            '0',
-            '@n',
-            '_',
-            '@n',
-            '!',
-            '@n',
-            '@',
-            '@n',
-            '#',
-            '@n',
-            '$',
-            '@n',
-            '%',
-            '@n',
-            '&',
-            '@n',
-            '*',
-            '@n',
+            "a",
+            "@n",
+            "j",
+            "@n",
+            "z",
+            "@n",
+            "A",
+            "@n",
+            "L",
+            "@n",
+            "Z",
+            "@n",
+            "1",
+            "@n",
+            "4",
+            "@n",
+            "7",
+            "@n",
+            "9",
+            "@n",
+            "0",
+            "@n",
+            "_",
+            "@n",
+            "!",
+            "@n",
+            "@",
+            "@n",
+            "#",
+            "@n",
+            "$",
+            "@n",
+            "%",
+            "@n",
+            "&",
+            "@n",
+            "*",
+            "@n",
         ]
         result = tokenizer.tokenize(test3)
         assert result == expected
 
         # Handles are allowed to precede the following characters
         test4 = "@n!a @n#a @n$a @n%a @n&a @n*a"
-        expected = ['!', 'a', '#', 'a', '$', 'a', '%', 'a', '&', 'a', '*', 'a']
+        expected = ["!", "a", "#", "a", "$", "a", "%", "a", "&", "a", "*", "a"]
         result = tokenizer.tokenize(test4)
         assert result == expected
 
         # Tests interactions with special symbols and multiple @
         test5 = "@n!@n @n#@n @n$@n @n%@n @n&@n @n*@n @n@n @@n @n@@n @n_@n @n7@n @nj@n"
         expected = [
-            '!',
-            '@n',
-            '#',
-            '@n',
-            '$',
-            '@n',
-            '%',
-            '@n',
-            '&',
-            '@n',
-            '*',
-            '@n',
-            '@n',
-            '@n',
-            '@',
-            '@n',
-            '@n',
-            '@',
-            '@n',
-            '@n_',
-            '@n',
-            '@n7',
-            '@n',
-            '@nj',
-            '@n',
+            "!",
+            "@n",
+            "#",
+            "@n",
+            "$",
+            "@n",
+            "%",
+            "@n",
+            "&",
+            "@n",
+            "*",
+            "@n",
+            "@n",
+            "@n",
+            "@",
+            "@n",
+            "@n",
+            "@",
+            "@n",
+            "@n_",
+            "@n",
+            "@n7",
+            "@n",
+            "@nj",
+            "@n",
         ]
         result = tokenizer.tokenize(test5)
         assert result == expected
 
         # Tests that handles can have a max length of 20
         test6 = "@abcdefghijklmnopqrstuvwxyz @abcdefghijklmnopqrst1234 @abcdefghijklmnopqrst_ @abcdefghijklmnopqrstendofhandle"
-        expected = ['uvwxyz', '1234', '_', 'endofhandle']
+        expected = ["uvwxyz", "1234", "_", "endofhandle"]
         result = tokenizer.tokenize(test6)
         assert result == expected
 
         # Edge case where an @ comes directly after a long handle
         test7 = "@abcdefghijklmnopqrstu@abcde @abcdefghijklmnopqrst@abcde @abcdefghijklmnopqrst_@abcde @abcdefghijklmnopqrst5@abcde"
         expected = [
-            'u',
-            '@abcde',
-            '@abcdefghijklmnopqrst',
-            '@abcde',
-            '_',
-            '@abcde',
-            '5',
-            '@abcde',
+            "u",
+            "@abcde",
+            "@abcdefghijklmnopqrst",
+            "@abcde",
+            "_",
+            "@abcde",
+            "5",
+            "@abcde",
         ]
         result = tokenizer.tokenize(test7)
         assert result == expected
@@ -343,7 +365,7 @@ class TestTokenize:
         assert result == expected
 
         # Test case with double quotation
-        test2 = "The DUP is similar to the \"religious right\" in the United States and takes a hardline stance on social issues"
+        test2 = 'The DUP is similar to the "religious right" in the United States and takes a hardline stance on social issues'
         expected = [
             (0, 3),
             (4, 7),
@@ -408,20 +430,35 @@ class TestTokenize:
         """
 
         sentence = "The 'v', I've been fooled but I'll seek revenge."
-        expected = ['The', "'", 'v', "'", ',', 'I', "'ve", 'been', 'fooled',
-                    'but', 'I', "'ll", 'seek', 'revenge', '.']
+        expected = [
+            "The",
+            "'",
+            "v",
+            "'",
+            ",",
+            "I",
+            "'ve",
+            "been",
+            "fooled",
+            "but",
+            "I",
+            "'ll",
+            "seek",
+            "revenge",
+            ".",
+        ]
         assert word_tokenize(sentence) == expected
 
         sentence = "'v' 're'"
-        expected = ["'", 'v', "'", "'re", "'"]
+        expected = ["'", "v", "'", "'re", "'"]
         assert word_tokenize(sentence) == expected
 
     def test_punkt_pair_iter(self):
 
         test_cases = [
-            ('12', [('1', '2'), ('2', None)]),
-            ('123', [('1', '2'), ('2', '3'), ('3', None)]),
-            ('1234', [('1', '2'), ('2', '3'), ('3', '4'), ('4', None)]),
+            ("12", [("1", "2"), ("2", None)]),
+            ("123", [("1", "2"), ("2", "3"), ("3", None)]),
+            ("1234", [("1", "2"), ("2", "3"), ("3", "4"), ("4", None)]),
         ]
 
         for (test_input, expected_output) in test_cases:
@@ -446,18 +483,23 @@ class TestTokenize:
 
         obj._lang_vars = TestPunktTokenizeWordsMock()
         # unpack generator, ensure that no error is raised
-        list(obj._tokenize_words('test'))
+        list(obj._tokenize_words("test"))
 
     def test_punkt_tokenize_custom_lang_vars(self):
 
         # Create LangVars including a full stop end character as used in Bengali
         class BengaliLanguageVars(punkt.PunktLanguageVars):
-            sent_end_chars = ('.', '?', '!', '\u0964')
-        obj = punkt.PunktSentenceTokenizer(lang_vars = BengaliLanguageVars())
+            sent_end_chars = (".", "?", "!", "\u0964")
+
+        obj = punkt.PunktSentenceTokenizer(lang_vars=BengaliLanguageVars())
 
         # We now expect these sentences to be split up into the individual sentences
         sentences = u"উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
-        expected = ["উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন।", "অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন।", "এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"]
+        expected = [
+            "উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন।",
+            "অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন।",
+            "এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।",
+        ]
 
         assert obj.tokenize(sentences) == expected
 
@@ -467,6 +509,48 @@ class TestTokenize:
 
         # We expect these sentences to not be split properly, as the Bengali full stop '।' is not included in the default language vars
         sentences = u"উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
-        expected = ["উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"]
+        expected = [
+            "উপরাষ্ট্রপতি শ্রী এম ভেঙ্কাইয়া নাইডু সোমবার আই আই টি দিল্লির হীরক জয়ন্তী উদযাপনের উদ্বোধন করেছেন। অনলাইনের মাধ্যমে এই অনুষ্ঠানে কেন্দ্রীয় মানব সম্পদ উন্নয়নমন্ত্রী শ্রী রমেশ পোখরিয়াল ‘নিশাঙ্ক’  উপস্থিত ছিলেন। এই উপলক্ষ্যে উপরাষ্ট্রপতি হীরকজয়ন্তীর লোগো এবং ২০৩০-এর জন্য প্রতিষ্ঠানের লক্ষ্য ও পরিকল্পনার নথি প্রকাশ করেছেন।"
+        ]
 
         assert obj.tokenize(sentences) == expected
+
+    @pytest.mark.parametrize(
+        "input_text,n_sents,n_splits,lang_vars",
+        [
+            # Test debug_decisions on a text with two sentences, split by a dot.
+            ("Subject: Some subject. Attachments: Some attachments", 2, 1),
+            # The sentence should be split into two sections,
+            # with one split and hence one decision.
+            # Test debug_decisions on a text with two sentences, split by an exclamation mark.
+            ("Subject: Some subject! Attachments: Some attachments", 2, 1),
+            # The sentence should be split into two sections,
+            # with one split and hence one decision.
+            # Test debug_decisions on a text with one sentences,
+            # which is not split.
+            ("This is just a normal sentence, just like any other.", 1, 0)
+            # Hence just 1
+        ],
+    )
+    def punkt_debug_decisions(self, input_text, n_sents, n_splits, lang_vars=None):
+        tokenizer = punkt.PunktSentenceTokenizer()
+        if lang_vars != None:
+            tokenizer._lang_vars = lang_vars
+
+        assert len(tokenizer.tokenize(input_text)) == n_sents
+        assert len(list(tokenizer.debug_decisions(input_text))) == n_splits
+
+    def test_punkt_debug_decisions_custom_end(self):
+        # Test debug_decisions on a text with two sentences,
+        # split by a custom end character, based on Issue #2519
+        class ExtLangVars(punkt.PunktLanguageVars):
+            sent_end_chars = (".", "?", "!", "^")
+
+        self.punkt_debug_decisions(
+            "Subject: Some subject^ Attachments: Some attachments",
+            n_sents=2,
+            n_splits=1,
+            lang_vars=ExtLangVars(),
+        )
+        # The sentence should be split into two sections,
+        # with one split and hence one decision.