Repository: cltk/cltk

Issue:
Greek not among word tokenizers
Hi! I attempted to use the Greek word tokenizer for a really quick search of a plain text file, as follows:

<pre>
from cltk.tokenize.word import WordTokenizer

word_tokenizer = WordTokenizer('greek')

for word in word_tokenizer.tokenize('files/plain_text/1-23_1.1-42.txt'):
	if word.endswith('εαι'):
		print(word)
</pre>

When I tried to call the script, I got the following error:

`AssertionError: Specific tokenizer not available for 'greek'. Only available for: '['arabic', 'latin', 'french', 'old_norse']'.`

Thanks!


Discussion:

Resolving patch:
diff --git a/cltk/corpus/swadesh.py b/cltk/corpus/swadesh.py
--- a/cltk/corpus/swadesh.py
+++ b/cltk/corpus/swadesh.py
@@ -22,7 +22,6 @@
 
 swadesh_syc=['ܐܢܐ‎','ܐܢܬ‎, ܐܢܬܝ‎', 'ܗܘ‎', 'ܚܢܢ‎,, ܐܢܚܢܢ‎', 'ܐܢܬܘܢ‎ , ܐܢܬܝܢ‎ ', 'ܗܢܘܢ‎ , ܗܢܝܢ‎', 'ܗܢܐ‎, ܗܕܐ‎', 'ܗܘ‎, ܗܝ‎', 'ܗܪܟܐ‎', 'ܬܡܢ‎', 'ܡܢ‎', 'ܡܐ‎, ܡܢ‎, ܡܢܐ‎, ܡܘܢ‎', 'ܐܝܟܐ‎', 'ܐܡܬܝ‎', 'ܐܝܟܢ‎,, ܐܝܟܢܐ‎', 'ܠܐ‎', 'ܟܠ‎', 'ܣܓܝ‎	', 'ܟܡܐ‎	', 'ܒܨܝܪܐ‎', 'ܐܚܪܢܐ‎, ܐܚܪܬܐ‎', 'ܚܕ‎ , ܚܕܐ‎', 'ܬܪܝܢ‎, ܬܪܬܝܢ‎', 'ܬܠܬܐ‎, ܬܠܬ‎', 'ܐܪܒܥܐ‎, ܐܪܒܥ‎', 'ܚܡܫܐ‎, ܚܡܫ‎', 'ܪܒܐ‎, ܟܒܝܪܐ‎	', 'ܐܪܝܟܐ‎', 'ܪܘܝܚܐ‎, ܦܬܝܐ‎', 'ܥܒܝܛܐ‎', 'ܢܛܝܠܐ‎, ܝܩܘܪܐ‎	', 'ܙܥܘܪܐ‎', 'ܟܪܝܐ‎', 'ܥܝܩܐ‎', 'ܪܩܝܩܐ‎, ܛܠܝܚܐ‎', 'ܐܢܬܬܐ‎', 'ܓܒܪܐ‎', 'ܐܢܫܐ‎', 'ܝܠܘܕܐ‎', 'ܐܢܬܬܐ‎', 'ܒܥܠܐ‎', 'ܐܡܐ‎', 'ܐܒܐ‎', 'ܚܝܘܬܐ‎', 'ܢܘܢܐ‎', 'ܛܝܪܐ‎, ܨܦܪܐ‎', 'ܟܠܒܐ‎', 'ܩܠܡܐ‎', 'ܚܘܝܐ‎', 'ܬܘܠܥܐ‎', 'ܐܝܠܢܐ‎', 'ܥܒܐ‎', 'ܩܝܣܐ‎', 'ܦܐܪܐ‎', 'ܙܪܥܐ‎', 'ܛܪܦܐ‎', 'ܫܪܫܐ‎	', 'ܩܠܦܬܐ‎', 'ܗܒܒܐ‎', 'ܓܠܐ‎', 'ܚܒܠܐ‎', 'ܓܠܕܐ‎	', 'ܒܣܪܐ‎', 'ܕܡܐ‎', 'ܓܪܡܐ‎', 'ܕܗܢܐ‎, ܫܘܡܢܐ‎', 'ܒܝܥܬܐ‎', 'ܩܪܢܐ‎', 'ܕܘܢܒܐ‎', 'ܐܒܪܐ‎', 'ܣܥܪܐ‎', 'ܪܝܫܐ‎', 'ܐܕܢܐ‎', 'ܥܝܢܐ‎', 'ܢܚܝܪܐ‎	', 'ܦܘܡܐ‎', 'ܫܢܐ‎, ܟܟܐ‎', 'ܠܫܢܐ‎', 'ܛܦܪܐ‎	', 'ܥܩܠܐ‎', 'ܪܓܠܐ‎', 'ܒܘܪܟܐ‎', 'ܐܝܕܐ‎', 'ܟܢܦܐ‎	', 'ܒܛܢܐ‎, ܟܪܣܐ‎	', 'ܡܥܝܐ‎, ܓܘܐ‎', 'ܨܘܪܐ‎, ܩܕܠܐ‎', 'ܚܨܐ‎, ܒܣܬܪܐ‎', 'ܚܕܝܐ‎', 'ܠܒܐ‎', 'ܟܒܕܐ‎', 'ܫܬܐ‎', 'ܐܟܠ‎', 'ܢܟܬ‎', 'ܡܨ‎	', 'ܪܩ‎', 'ܓܥܛ‎', 'ܢܦܚ‎', 'ܢܦܫ‎, ܢܫܡ‎', 'ܓܚܟ‎	', 'ܚܙܐ‎', 'ܫܡܥ‎', 'ܝܕܥ‎', 'ܚܫܒ‎', 'ܡܚ‎, ܣܩ‎', 'ܕܚܠ‎, ܟܘܪ‎', 'ܕܡܟ‎', 'ܚܝܐ‎	', 'ܡܝܬ‎', 'ܩܛܠ‎', 'ܟܬܫ‎', 'ܨܝܕ‎	', 'ܡܚܐ‎, ܢܩܫ‎', 'ܓܕܡ‎, ܩܛܥ‎', 'ܫܪܩ‎, ܦܕܥ‎, ܦܪܬ‎', 'ܕܓܫ‎', 'ܚܟ‎, ܣܪܛ‎', 'ܚܦܪ‎', 'ܣܚܐ‎', 'ܦܪܚ‎	', 'ܗܠܟ‎	', 'ܐܬܐ‎	', 'ܫܟܒ‎, ܡܟ‎', 'ܝܬܒ‎', 'ܬܪܨ‎', 'ܦܢܐ‎, ܥܛܦ‎	', 'ܢܦܠ‎', 'ܝܗܒ‎, ܢܬܠ‎', 'ܐܚܕ‎', 'ܩܡܛ‎, ܥܨܪ‎', 'ܫܦ‎, ܚܟ‎', 'ܚܠܠ‎, ܦܝܥ‎', 'ܟܦܪ‎', 'ܓܪܫ‎', 'ܙܥܦ‎	', 'ܪܡܐ‎', 'ܐܣܪ‎, ܩܛܪ‎', 'ܚܝܛ‎', 'ܡܢܐ‎', 'ܐܡܪ‎', 'ܙܡܪ‎', 'ܫܥܐ‎', 'ܛܦ‎', 'ܪܣܡ‎, ܫܚܠ‎', 'ܓܠܕ‎, ܩܪܫ‎', 'ܙܘܐ‎, ܥܒܐ‎', 'ܫܡܫܐ‎', 'ܣܗܪܐ‎', 'ܟܘܟܒܐ‎', 'ܡܝܐ‎	', 'ܡܛܪܐ‎', 'ܢܗܪܐ‎', 'ܝܡܬܐ‎', 'ܝܡܐ‎', 'ܡܠܚܐ‎	', 'ܟܐܦܐ‎, ܐܒܢܐ‎, ܫܘܥܐ‎', 'ܚܠܐ‎', 'ܐܒܩܐ‎, ܕܩܬܐ‎', 'ܐܪܥܐ‎', 'ܥܢܢܐ‎, ܥܝܡܐ‎, ܥܝܒܐ‎', 'ܥܪܦܠܐ‎	', 'ܫܡܝܐ‎', 'ܪܘܚܐ‎	', 'ܬܠܓܐ‎', 'ܓܠܝܕܐ‎', 'ܬܢܢܐ‎	', 'ܢܘܪܐ‎, ܐܫܬܐ‎', 'ܩܛܡܐ‎	', 'ܝܩܕ‎', 'ܐܘܪܚܐ‎', 'ܛܘܪܐ‎', 'ܣܘܡܩܐ‎', 'ܝܘܪܩܐ‎', 'ܫܥܘܬܐ‎', 'ܚܘܪܐ‎', 'ܐܘܟܡܐ‎	', 'ܠܠܝܐ‎	', 'ܝܘܡܐ‎	', 'ܫܢܬܐ‎', 'ܫܚܝܢܐ‎', 'ܩܪܝܪܐ‎', 'ܡܠܝܐ‎', 'ܚܕܬܐ‎', 'ܥܬܝܩܐ‎', 'ܛܒܐ‎', 'ܒܝܫܐ‎', 'ܒܩܝܩܐ‎ ܚܪܝܒܐ‎', 'ܫܘܚܬܢܐ‎', 'ܬܪܝܨܐ‎	', 'ܚܘܕܪܢܝܐ‎', 'ܚܪܝܦܐ‎', 'ܩܗܝܐ‎', 'ܦܫܝܩܐ‎', 'ܪܛܝܒܐ‎, ܬܠܝܠܐ‎', 'ܝܒܝܫܐ‎', 'ܬܪܝܨܐ‎	', 'ܩܪܝܒܐ‎', 'ܪܚܝܩܐ‎', 'ܝܡܝܢܐ‎', 'ܣܡܠܐ‎', 'ܒ-‎, ܠܘܬ‎', 'ܥܡ‎', 'ܐܢ‎', '-ܡܛܠ ܕ‎, ܒܥܠܬ‎', 'ܫܡܐ‎']
 
-
 class Swadesh():
     def __init__(self, language):
         self.language = language
diff --git a/cltk/tokenize/indian_tokenizer.py b/cltk/tokenize/indian_tokenizer.py
deleted file mode 100644
--- a/cltk/tokenize/indian_tokenizer.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""Tokenizer for Indian languages/scripts."""
-
-import re
-import string
-
-__author__ = ['Anoop Kunchukuttan']
-__copyright = 'GPL'
-
-modified_punctuations = string.punctuation.replace("|","") # The replace , deletes the ' | ' from the punctuation string provided by the library
-
-indian_punctuation_pattern = re.compile('(['+modified_punctuations+'\u0964\u0965'+']|\|+)')
-
-
-def indian_punctuation_tokenize_regex(input_str):
-    """A trivial tokenizer which just tokenizes on the punctuation boundaries.
-    This also includes punctuation, namely the the purna virama ("|") and
-    deergha virama ("॥"), for Indian language scripts.
-
-    >>> indian_str = "प्रेमचन्द का जन्म ३१ जुलाई सन् १८८० को बनारस शहर।"
-    >>> indian_punctuation_tokenize_regex(indian_str)
-    ['प्रेमचन्द', 'का', 'जन्म', '३१', 'जुलाई', 'सन्', '१८८०', 'को', 'बनारस', 'शहर', '।']
-
-    :param input_str: A string to be
-    :type input_str: string
-    :return List of word tokens.
-    :rtype: list
-    """
-    tok_str = indian_punctuation_pattern.sub(r' \1 ',input_str.replace('\t',' '))
-    return re.sub(r'[ ]+',u' ',tok_str).strip(' ').split(' ')
-
-
-if __name__ == '__main__':
-    example = indian_punctuation_tokenize_regex("हिन्दी भारत की सबसे अधिक बोली और समझी जाने वाली भाषा है।")
-    print(example)
diff --git a/cltk/tokenize/sentence.py b/cltk/tokenize/sentence.py
--- a/cltk/tokenize/sentence.py
+++ b/cltk/tokenize/sentence.py
@@ -1,6 +1,6 @@
 """Tokenize sentences."""
 
-__author__ = ['Kyle P. Johnson <kyle@kyle-p-johnson.com>']
+__author__ = ['Kyle P. Johnson <kyle@kyle-p-johnson.com>','Anoop Kunchukuttan']
 __license__ = 'MIT License. See LICENSE.'
 
 
@@ -8,6 +8,8 @@
 from nltk.tokenize.punkt import PunktLanguageVars
 from nltk.tokenize.punkt import PunktSentenceTokenizer
 import os
+import re
+import string
 
 
 PUNCTUATION = {'greek':
@@ -19,6 +21,7 @@
                     'internal': (',', ';'),
                     'file': 'latin.pickle', }}
 
+INDIAN_LANGUAGES = ['bengali','hindi','marathi','sanskrit','telugu']
 
 class TokenizeSentence():  # pylint: disable=R0903
     """Tokenize sentences for the language given as argument, e.g.,
@@ -31,8 +34,10 @@ def __init__(self: object, language: str):
         :param language : Language for sentence tokenization.
         """
         self.language = language.lower()
-        self.internal_punctuation, self.external_punctuation, self.tokenizer_path = \
-            self._setup_language_variables(self.language)
+
+        if self.language not in INDIAN_LANGUAGES :
+            self.internal_punctuation, self.external_punctuation, self.tokenizer_path = \
+                self._setup_language_variables(self.language)
 
     def _setup_language_variables(self, lang: str):
         """Check for language availability and presence of tokenizer file,
@@ -88,9 +93,26 @@ def tokenize_sentences(self: object, untokenized_string: str):
         for sentence in tokenizer.sentences_from_text(untokenized_string, realign_boundaries=True):  # pylint: disable=C0301
             tokenized_sentences.append(sentence)
         return tokenized_sentences
-        
+
+    def indian_punctuation_tokenize_regex(self: object, untokenized_string: str):
+        """A trivial tokenizer which just tokenizes on the punctuation boundaries.
+        This also includes punctuation, namely the the purna virama ("|") and
+        deergha virama ("॥"), for Indian language scripts.
+
+        :type untokenized_string: str
+        :param untokenized_string: A string containing one of more sentences.
+        :rtype : list of strings
+        """
+        modified_punctuations = string.punctuation.replace("|","") # The replace , deletes the ' | ' from the punctuation string provided by the library
+        indian_punctuation_pattern = re.compile('(['+modified_punctuations+'\u0964\u0965'+']|\|+)')
+        tok_str = indian_punctuation_pattern.sub(r' \1 ',untokenized_string.replace('\t',' '))
+        return re.sub(r'[ ]+',u' ',tok_str).strip(' ').split(' ')
+
     def tokenize(self: object, untokenized_string: str):
         # NLTK's PlaintextCorpusReader needs a function called tokenize
         # in functions used as a parameter for sentence tokenization.
         # So this is an alias for tokenize_sentences().
-        return self.tokenize_sentences(untokenized_string)
+        if self.language in INDIAN_LANGUAGES:
+            return self.indian_punctuation_tokenize_regex(untokenized_string)
+        else:
+            return self.tokenize_sentences(untokenized_string)


Test patch:
diff --git a/cltk/tests/test_stop.py b/cltk/tests/test_stop.py
--- a/cltk/tests/test_stop.py
+++ b/cltk/tests/test_stop.py
@@ -11,7 +11,7 @@
 from cltk.stop.classical_hindi.stops import STOPS_LIST as HINDI_STOPS
 from cltk.stop.arabic.stopword_filter import stopwords_filter as arabic_stop_filter
 from cltk.stop.old_norse.stops import STOPS_LIST as OLD_NORSE_STOPS
-from cltk.tokenize.indian_tokenizer import indian_punctuation_tokenize_regex
+from cltk.tokenize.sentence import TokenizeSentence
 from nltk.tokenize.punkt import PunktLanguageVars
 from cltk.tokenize.word import WordTokenizer
 import os
@@ -122,7 +122,8 @@ def test_classical_hindi_stops(self):
         Sentence extracted from (https://github.com/cltk/hindi_text_ltrc/blob/master/miscellaneous/gandhi/main.txt)
         """
         sentence = " वह काबुली फिर वहां आकर खडा हो गया है  "
-        tokens = indian_punctuation_tokenize_regex(sentence)
+        tokenizer = TokenizeSentence('hindi')
+        tokens = tokenizer.tokenize(sentence)
         no_stops = [word for word in tokens if word not in HINDI_STOPS]
         target_list = ['काबुली', 'फिर', 'वहां', 'आकर', 'खडा', 'गया']
         self.assertEqual(no_stops, target_list)