Repository: cltk/cltk

Issue:
Greek not among word tokenizers
Hi! I attempted to use the Greek word tokenizer for a really quick search of a plain text file, as follows:

<pre>
from cltk.tokenize.word import WordTokenizer

word_tokenizer = WordTokenizer('greek')

for word in word_tokenizer.tokenize('files/plain_text/1-23_1.1-42.txt'):
	if word.endswith('εαι'):
		print(word)
</pre>

When I tried to call the script, I got the following error:

`AssertionError: Specific tokenizer not available for 'greek'. Only available for: '['arabic', 'latin', 'french', 'old_norse']'.`

Thanks!


Discussion:

Resolving patch:
diff --git a/cltk/corpus/swadesh.py b/cltk/corpus/swadesh.py
--- a/cltk/corpus/swadesh.py
+++ b/cltk/corpus/swadesh.py
@@ -76,6 +76,7 @@
               'अच्छा', 'बुरा', 'सड़ा', 'गन्दा', 'सीधा', 'गोल', 'तीखा', 'कुंद', 'चिकना', 'गीला', 'सूखा', 'सही', 'नज़दीक',
               'दूर', 'दायाँ', 'बायाँ', 'पे', 'में', 'के साथ', 'और', 'अगर', 'क्योंकि', 'नाम']
 
+
 swadesh_la = ['ego', 'tū', 'is, ea, id', 'nōs', 'vōs', 'eī, iī, eae, ea', 'hic, haec, hoc', 'ille, illa, illud', 'hīc',
               'illic, ibi', 'quis, quae', 'quid', 'ubi', 'cum', 'quōmodō', 'nōn, nē', 'omnēs, omnia',
               'multī, multae, multa', 'aliquī, aliqua, aliquod', 'paucī, paucae, pauca', 'alter, alius', 'ūnus', 'duō',
@@ -100,6 +101,7 @@
               'humidus, aqueus', 'siccus', 'rectus', 'propinquus, proximus', 'longus', 'dexter', 'laevus, sinister',
               'ad, in', 'in', 'cum', 'et, -que', 'si', 'quod', 'nōmen']
 
+
 swadesh_old_norse = ["ek", "þú", "hann", "vér", "þér", "þeir", "sjá, þessi", "sá", "hér", "þar", "hvar", "hvat", "hvar",
                      "hvenær", "hvé", "eigi", "allr", "margr", "nǫkkurr", "fár", "annarr", "einn", "tveir", "þrír",
                      "fjórir", "fimm", "stórr", "langr", "breiðr", "þykkr", "þungr", "lítill", "stuttr", "mjór",
diff --git a/cltk/corpus/telugu/alphabet.py b/cltk/corpus/telugu/alphabet.py
new file mode 100644
--- /dev/null
+++ b/cltk/corpus/telugu/alphabet.py
@@ -0,0 +1,16 @@
+"""Telugu alphabet"""
+__author__ = 'Nishchith Shetty <inishchith[at]gmail[dot]com>'
+
+VOWELS = ['అ ','ఆ','ఇ','ఈ ','ఉ ','ఊ ','ఋ  ','ౠ  ','ఌ ',
+          'ౡ','ఎ','ఏ','ఐ','ఒ','ఓ','ఔ ','అం','అః']
+
+CONSONANTS = ['క', 'ఖ','గ','ఘ','ఙ'
+              'చ','ఛ','జ','ఝ','ఞ',
+              'ట','ఠ','డ ','ఢ','ణ',
+              'త','థ','ద','ధ','న',
+              'ప','ఫ','బ','భ','మ',
+              'య','ర','ల','వ',
+              'శ','ష','స ','హ',
+              'ళ','క్ష ','ఱ']
+
+NUMERLALS = ['౦','౧','౨','౩','౪	','౫','౬','౭','౮','౯']


Test patch:
diff --git a/cltk/tests/test_nlp/test_tokenize.py b/cltk/tests/test_nlp/test_tokenize.py
--- a/cltk/tests/test_nlp/test_tokenize.py
+++ b/cltk/tests/test_nlp/test_tokenize.py
@@ -1,7 +1,6 @@
 # -*-coding:utf-8-*-
 """Test cltk.tokenize.
 
-TODO: Add tests for the Indian lang tokenizers: from cltk.tokenize.indian_tokenizer import indian_punctuation_tokenize_regex
 """
 
 from cltk.corpus.utils.importer import CorpusImporter
@@ -249,6 +248,45 @@ def test_middle_high_german_tokenizer(self):
         tokenized_lines = tokenizer.tokenize(text)
         self.assertTrue(tokenized_lines == target)
 
+    def test_sentence_tokenizer_bengali(self):
+        """Test tokenizing bengali sentences."""
+        text = "দুর্ব্বাসার শাপে রাজা শকুন্তলাকে একেবারে ভুলে বেশ সুখে আছেন।"
+        target = ['দুর্ব্বাসার', 'শাপে', 'রাজা', 'শকুন্তলাকে', 'একেবারে', 'ভুলে', 'বেশ', 'সুখে', 'আছেন', '।']
+        tokenizer = TokenizeSentence('bengali')
+        tokenized_sentences = tokenizer.tokenize(text)
+        self.assertEqual(tokenized_sentences, target)
+
+    def test_sentence_tokenizer_classical_hindi(self):
+        """Test tokenizing classical_hindi sentences."""
+        text = "जलर्  चिकित्सा से उन्हें कोई लाभ नहीं हुआ।"
+        target = ['जलर्', 'चिकित्सा', 'से', 'उन्हें', 'कोई', 'लाभ', 'नहीं', 'हुआ', '।']
+        tokenizer = TokenizeSentence('hindi')
+        tokenized_sentences = tokenizer.tokenize(text)
+        self.assertEqual(tokenized_sentences, target)
+
+    def test_sentence_tokenizer_marathi(self):
+        """Test tokenizing marathi sentences."""
+        text = "अर्जुन उवाच । एवं सतत युक्ता ये भक्तास्त्वां पर्युपासते । ये चाप्यक्षरमव्यक्तं तेषां के योगवित्तमाः ॥"
+        target = ['अर्जुन', 'उवाच', '।', 'एवं', 'सतत', 'युक्ता', 'ये', 'भक्तास्त्वां', 'पर्युपासते', '।', 'ये', 'चाप्यक्षरमव्यक्तं', 'तेषां', 'के', 'योगवित्तमाः', '॥']
+        tokenizer = TokenizeSentence('marathi')
+        tokenized_sentences = tokenizer.tokenize(text)
+        self.assertEqual(tokenized_sentences, target)
+
+    def test_sentence_tokenizer_sanskrit(self):
+        """Test tokenizing sanskrit sentences."""
+        text = "श्री भगवानुवाच पश्य मे पार्थ रूपाणि शतशोऽथ सहस्रशः। नानाविधानि दिव्यानि नानावर्णाकृतीनि च।।"
+        target = ['श्री', 'भगवानुवाच', 'पश्य', 'मे', 'पार्थ', 'रूपाणि', 'शतशोऽथ', 'सहस्रशः', '।', 'नानाविधानि', 'दिव्यानि', 'नानावर्णाकृतीनि', 'च', '।', '।']
+        tokenizer = TokenizeSentence('sanskrit')
+        tokenized_sentences = tokenizer.tokenize(text)
+        self.assertEqual(tokenized_sentences, target)
+
+    def test_sentence_tokenizer_telugu(self):
+        """Test tokenizing telugu sentences."""
+        text = "తా. ఎక్కడెక్కడ బుట్టిన నదులును రత్నాకరుడను నాశతో సముద్రుని చేరువిధముగా నెన్నియిక్కట్టులకైన నోర్చి ప్రజలు దమంతట దామె ప్రియముం జూపుచు ధనికుని యింటికేతెంచుచుందురు."
+        target = ['తా', '.', 'ఎక్కడెక్కడ', 'బుట్టిన', 'నదులును', 'రత్నాకరుడను', 'నాశతో', 'సముద్రుని', 'చేరువిధముగా', 'నెన్నియిక్కట్టులకైన', 'నోర్చి', 'ప్రజలు', 'దమంతట', 'దామె', 'ప్రియముం', 'జూపుచు', 'ధనికుని', 'యింటికేతెంచుచుందురు', '.']
+        tokenizer = TokenizeSentence('telugu')
+        tokenized_sentences = tokenizer.tokenize(text)
+        self.assertEqual(tokenized_sentences, target)
     def test_akkadian_word_tokenizer(self):
         """
         Tests word_tokenizer.