Repository: cltk/cltk

Issue:
Latin sentence tokenizer splits on limited punctuation
The current version of the Latin punkt sentence tokenizer only splits on ('.', '?', '!'). More flexibility is needed to account for editorial practice, digitization choices, etc.

re: editorial practice, cf. e.g. discussion in Pinkster, H. 2010. “Notes on the Language of Marcus Caelius Rufus.” In Dickey, E. and Chahoud, A. eds. Colloquial and Literary Latin. Cambridge: Cambridge University Press; page 195-196. https://books.google.com/books?id=iXlEtbSauCYC&pg=PA196#v=onepage&q&f=false.

re: digitization choices, cf.—at an extreme—the punctuation of the the Tesserae version of the Vulgate (https://github.com/cltk/latin_text_tesserae/blob/master/texts/jerome.vulgate.part.1.genesis.tess) which only uses semicolons.


Discussion:

Resolving patch:
diff --git a/cltk/tokenize/latin/params.py b/cltk/tokenize/latin/params.py
--- a/cltk/tokenize/latin/params.py
+++ b/cltk/tokenize/latin/params.py
@@ -154,3 +154,6 @@
 
 class LatinLanguageVars(PunktLanguageVars):
     _re_non_word_chars = PunktLanguageVars._re_non_word_chars.replace("'",'')
+
+PUNCTUATION = ('.', '?', '!')
+STRICT_PUNCTUATION = PUNCTUATION+('-', ':', ';')
diff --git a/cltk/tokenize/latin/sentence.py b/cltk/tokenize/latin/sentence.py
--- a/cltk/tokenize/latin/sentence.py
+++ b/cltk/tokenize/latin/sentence.py
@@ -5,28 +5,32 @@
 __license__ = 'MIT License.'
 
 import os.path
-
+import nltk
+from nltk.tokenize.punkt import PunktLanguageVars
 from cltk.tokenize.sentence import BaseSentenceTokenizer, BasePunktSentenceTokenizer
-from cltk.tokenize.latin.params import LatinLanguageVars
+from cltk.tokenize.latin.params import LatinLanguageVars, PUNCTUATION, STRICT_PUNCTUATION
 from cltk.utils.file_operations import open_pickle
 
-def SentenceTokenizer(tokenizer:str = 'punkt'):
+def SentenceTokenizer(tokenizer:str = 'punkt', strict:bool = False):
     if tokenizer=='punkt':
-        return LatinPunktSentenceTokenizer()
+        return LatinPunktSentenceTokenizer(strict=strict)
 
 
 class LatinPunktSentenceTokenizer(BasePunktSentenceTokenizer):
     """ PunktSentenceTokenizer trained on Latin
     """
     models_path = os.path.normpath(get_cltk_data_dir() + '/latin/model/latin_models_cltk/tokenizers/sentence')
-    missing_models_message = "BackoffLatinLemmatizer requires the ```latin_models_cltk``` to be in cltk_data. Please load this corpus."
+    missing_models_message = "LatinPunktSentenceTokenizer requires the ```latin_models_cltk``` to be in cltk_data. Please load this corpus."
 
-    def __init__(self: object, language:str = 'latin'):
+    def __init__(self: object, language:str = 'latin', strict:bool = False):
         """
         :param language : language for sentence tokenization
         :type language: str
+        :param strict : allow for stricter puctuation for sentence tokenization
+        :type strict: bool
         """
         self.lang_vars = LatinLanguageVars()
+        self.strict = strict
         super().__init__(language='latin', lang_vars=self.lang_vars)
         self.models_path = LatinPunktSentenceTokenizer.models_path
 
@@ -34,3 +38,8 @@ def __init__(self: object, language:str = 'latin'):
             self.model =  open_pickle(os.path.join(self.models_path, 'latin_punkt.pickle'))
         except FileNotFoundError as err:
             raise type(err)(LatinPunktSentenceTokenizer.missing_models_message)
+
+        if self.strict:
+            PunktLanguageVars.sent_end_chars=STRICT_PUNCTUATION
+        else:
+            PunktLanguageVars.sent_end_chars=PUNCTUATION
diff --git a/cltk/tokenize/sentence.py b/cltk/tokenize/sentence.py
--- a/cltk/tokenize/sentence.py
+++ b/cltk/tokenize/sentence.py
@@ -164,11 +164,3 @@ def tokenize(self, untokenized_string: str, model=None):
         :param untokenized_string: A string containing one of more sentences.
         """
         return self.tokenize_sentences(untokenized_string)
-
-if __name__ == "__main__":
-    text = """श्री भगवानुवाच भूय एव महाबाहो श्रृणु मे परमं वचः। यत्तेऽहं प्रीयमाणाय वक्ष्यामि हितकाम्यया।।
-न मे विदुः सुरगणाः प्रभवं न महर्षयः। अहमादिर्हि देवानां महर्षीणां च सर्वशः।।"""
-    t = TokenizeSentence('sanskrit')
-    sents = t.tokenize(text)
-    for i, sent in enumerate(sents, 1):
-        print(f'{i}: {sent}')


Test patch:
diff --git a/cltk/tests/test_nlp/test_tokenize.py b/cltk/tests/test_nlp/test_tokenize.py
--- a/cltk/tests/test_nlp/test_tokenize.py
+++ b/cltk/tests/test_nlp/test_tokenize.py
@@ -50,10 +50,16 @@ def test_sentence_tokenizer_latin_punkt(self):
                   'Hos ego video consul et de re publica sententiam rogo et, quos ferro trucidari oportebat, eos nondum voce volnero!',
                   'Fuisti igitur apud Laecam illa nocte, Catilina, distribuisti partes Italiae, statuisti, quo quemque proficisci placeret, delegisti, quos Romae relinqueres, quos tecum educeres, discripsisti urbis partes ad incendia, confirmasti te ipsum iam esse exiturum, dixisti paulum tibi esse etiam nunc morae, quod ego viverem.']  # pylint: disable=line-too-long
         tokenizer = LatinPunktSentenceTokenizer()
-        print(tokenizer.models_path)
         tokenized_sentences = tokenizer.tokenize(self.latin_text)
         self.assertEqual(tokenized_sentences, target)
 
+    def test_sentence_tokenizer_latin_punkt_strict(self):
+        """Test tokenizing Latin sentences with stricter punctuation."""
+        target = ['in principio creavit Deus caelum et terram;', 'terra autem erat inanis et vacua et tenebrae super faciem abyssi et spiritus Dei ferebatur super aquas;', 'dixitque Deus fiat lux et facta est lux;', 'et vidit Deus lucem quod esset bona et divisit lucem ac tenebras.']  # pylint: disable=line-too-long
+        tokenizer = LatinPunktSentenceTokenizer(strict=True)
+        tokenized_sentences = tokenizer.tokenize("""in principio creavit Deus caelum et terram; terra autem erat inanis et vacua et tenebrae super faciem abyssi et spiritus Dei ferebatur super aquas; dixitque Deus fiat lux et facta est lux; et vidit Deus lucem quod esset bona et divisit lucem ac tenebras.""")
+        self.assertEqual(tokenized_sentences, target)
+
     # Deprecated use cltk.tokenize.latin.sentence
     def test_sentence_tokenizer_latin(self):
         """Test tokenizing Latin sentences."""