Repository: cltk/cltk

Issue:
Greek not among word tokenizers
Hi! I attempted to use the Greek word tokenizer for a really quick search of a plain text file, as follows:

<pre>
from cltk.tokenize.word import WordTokenizer

word_tokenizer = WordTokenizer('greek')

for word in word_tokenizer.tokenize('files/plain_text/1-23_1.1-42.txt'):
	if word.endswith('εαι'):
		print(word)
</pre>

When I tried to call the script, I got the following error:

`AssertionError: Specific tokenizer not available for 'greek'. Only available for: '['arabic', 'latin', 'french', 'old_norse']'.`

Thanks!

Comments:

Patch:
diff --git a/cltk/lemmatize/latin/backoff.py b/cltk/lemmatize/latin/backoff.py
index ab5a3be2..46497c47 100755
--- a/cltk/lemmatize/latin/backoff.py
+++ b/cltk/lemmatize/latin/backoff.py
@@ -518,13 +518,14 @@ class BackoffLatinLemmatizer(object):
         self.pos_train_sents, self.train_sents, self.test_sents = _randomize_data(self.train, self.seed)
 
     def _define_lemmatizer(self):
+        # Suggested backoff chain--should be tested for optimal order
         backoff0 = None
         backoff1 = IdentityLemmatizer()
         backoff2 = TrainLemmatizer(model=self.LATIN_OLD_MODEL, backoff=backoff1)
         backoff3 = PPLemmatizer(regexps=self.latin_verb_patterns, pps=self.latin_pps, backoff=backoff2)                 
-        backoff4 = UnigramLemmatizer(self.train_sents, backoff=backoff3)        
-        backoff5 = RegexpLemmatizer(self.latin_sub_patterns, backoff=backoff4)
-        backoff6 = TrainLemmatizer(model=self.LATIN_MODEL, backoff=backoff5)        
+        backoff4 = RegexpLemmatizer(self.latin_sub_patterns, backoff=backoff3)
+        backoff5 = UnigramLemmatizer(self.train_sents, backoff=backoff4)
+        backoff6 = TrainLemmatizer(model=self.LATIN_MODEL, backoff=backoff5)      
         #backoff7 = BigramPOSLemmatizer(self.pos_train_sents, include=['cum'], backoff=backoff6)
         #lemmatizer = backoff7
         lemmatizer = backoff6
diff --git a/cltk/tokenize/word.py b/cltk/tokenize/word.py
index b1329109..34b2f160 100644
--- a/cltk/tokenize/word.py
+++ b/cltk/tokenize/word.py
@@ -8,6 +8,7 @@ from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktParameters
 
 import re
 
+# Cleanup these imports—most are not used!
 from nltk.data              import load
 from nltk.tokenize.casual   import (TweetTokenizer, casual_tokenize)
 from nltk.tokenize.mwe      import MWETokenizer
@@ -41,20 +42,28 @@ class WordTokenizer:  # pylint: disable=too-few-public-methods
         """Take language as argument to the class. Check availability and
         setup class variables."""
         self.language = language
-        self.available_languages = ['arabic', 'latin', 'french', 'old_norse']
+        self.available_languages = ['arabic', 
+                                    'french',
+                                    'greek',
+                                    'latin',
+                                    'old_norse']
         assert self.language in self.available_languages, \
             "Specific tokenizer not available for '{0}'. Only available for: '{1}'.".format(self.language,  # pylint: disable=line-too-long
-                                                                                            self.available_languages)  # pylint: disable=line-too-long
+            self.available_languages)  # pylint: disable=line-too-long
+        # ^^^ Necessary? since we have an 'else' in `tokenize`
+        
 
     def tokenize(self, string):
         """Tokenize incoming string."""
-
-        if self.language == 'latin':
-            tokens = tokenize_latin_words(string)
+        
+        if self.language == 'arabic':
+            tokens = tokenize_arabic_words(string)
         elif self.language == 'french':
             tokens = tokenize_french_words(string)
-        elif self.language == 'arabic':
-            tokens = tokenize_arabic_words(string)
+        elif self.language == 'greek':
+            tokens = tokenize_greek_words(string)
+        elif self.language == 'latin':
+            tokens = tokenize_latin_words(string)
         elif self.language == 'old_norse':
             tokens = tokenize_old_norse_words(string)
         else:
@@ -101,6 +110,56 @@ def nltk_tokenize_words(string, attached_period=False, language=None):
     return new_tokens
 
 
+def tokenize_arabic_words(text):
+
+    """
+        Tokenize text into words
+        @param text: the input text.
+        @type text: unicode.
+        @return: list of words.
+        @rtype: list.
+    """
+    specific_tokens = []
+    if not text:
+        return specific_tokens
+    else:
+        specific_tokens = araby.tokenize(text)
+        return specific_tokens
+        
+
+def tokenize_french_words(string):
+    assert isinstance(string, str), "Incoming string must be type str."
+
+    # normalize apostrophes
+
+    text = re.sub(r"’", r"'", string)
+
+    # Dealing with punctuation
+    text = re.sub(r"\'", r"' ", text)
+    text = re.sub("(?<=.)(?=[.!?)(\";:,«»\-])", " ", text)
+
+    results = str.split(text)
+    return (results)
+    
+    
+def tokenize_greek_words(text):
+    """
+    Tokenizer divides the string into a list of substrings. This is a placeholder
+    function that returns the default NLTK word tokenizer until
+    Greek-specific options are added.
+    
+    Example:
+    >>> text = 'Θουκυδίδης Ἀθηναῖος ξυνέγραψε τὸν πόλεμον τῶν Πελοποννησίων καὶ Ἀθηναίων,'
+    >>> tokenize_greek_words(text)
+    ['Θουκυδίδης', 'Ἀθηναῖος', 'ξυνέγραψε', 'τὸν', 'πόλεμον', 'τῶν', 'Πελοποννησίων', 'καὶ', 'Ἀθηναίων', ',']
+      
+    :param string: This accepts the string value that needs to be tokenized
+    :returns: A list of substrings extracted from the string
+    """
+    
+    return nltk_tokenize_words(text) # Simplest implementation to start
+        
+
 def tokenize_latin_words(string):
     """
     Tokenizer divides the string into a list of substrings
@@ -211,38 +270,6 @@ def tokenize_latin_words(string):
     return specific_tokens
 
 
-def tokenize_french_words(string):
-    assert isinstance(string, str), "Incoming string must be type str."
-
-    # normalize apostrophes
-
-    text = re.sub(r"’", r"'", string)
-
-    # Dealing with punctuation
-    text = re.sub(r"\'", r"' ", text)
-    text = re.sub("(?<=.)(?=[.!?)(\";:,«»\-])", " ", text)
-
-    results = str.split(text)
-    return (results)
-
-
-def tokenize_arabic_words(text):
-
-    """
-        Tokenize text into words
-        @param text: the input text.
-        @type text: unicode.
-        @return: list of words.
-        @rtype: list.
-    """
-    specific_tokens = []
-    if not text:
-        return specific_tokens
-    else:
-        specific_tokens = araby.tokenize(text)
-        return specific_tokens
-
-
 def tokenize_old_norse_words(text):
     """
 
diff --git a/docs/greek.rst b/docs/greek.rst
index 4ddcca68..9937650f 100644
--- a/docs/greek.rst
+++ b/docs/greek.rst
@@ -856,6 +856,20 @@ the Greek language. Currently, the only available dialect is Attic as reconstruc
    Out[3]: '[di.ó.tʰen kɑj dis.kɛ́ːp.trọː ti.mɛ̂ːs o.kʰy.ron zdêw.gos ɑ.trẹː.dɑ̂n stó.lon ɑr.gẹ́ː.ɔːn]'
 
 
+Word Tokenization
+=================
+
+.. code-block:: python
+
+   In [1]: from cltk.tokenize.word import WordTokenizer
+
+   In [2]: word_tokenizer = WordTokenizer('greek')
+
+   In [3]: text = 'Θουκυδίδης Ἀθηναῖος ξυνέγραψε τὸν πόλεμον τῶν Πελοποννησίων καὶ Ἀθηναίων,'
+
+   In [4]: word_tokenizer.tokenize(text)
+   Out[4]: ['Θουκυδίδης', 'Ἀθηναῖος', 'ξυνέγραψε', 'τὸν', 'πόλεμον', 'τῶν', 'Πελοποννησίων', 'καὶ', 'Ἀθηναίων', ',']
+   
 
 Word2Vec
 ========


Test patch:
diff --git a/cltk/tests/test_tokenize.py b/cltk/tests/test_tokenize.py
index 668292ee..c5856900 100644
--- a/cltk/tests/test_tokenize.py
+++ b/cltk/tests/test_tokenize.py
@@ -61,6 +61,23 @@ class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
         self.assertEqual(len(tokenized_sentences), len(good_tokenized_sentences))
     '''
 
+
+    def test_greek_word_tokenizer(self):
+        """Test Latin-specific word tokenizer."""
+        word_tokenizer = WordTokenizer('greek')
+        
+        # Test sources:
+        # - Thuc. 1.1.1       
+        
+        test = "Θουκυδίδης Ἀθηναῖος ξυνέγραψε τὸν πόλεμον τῶν Πελοποννησίων καὶ Ἀθηναίων, ὡς ἐπολέμησαν πρὸς ἀλλήλους, ἀρξάμενος εὐθὺς καθισταμένου καὶ ἐλπίσας μέγαν τε ἔσεσθαι καὶ ἀξιολογώτατον τῶν προγεγενημένων, τεκμαιρόμενος ὅτι ἀκμάζοντές τε ᾖσαν ἐς αὐτὸν ἀμφότεροι παρασκευῇ τῇ πάσῃ καὶ τὸ ἄλλο Ἑλληνικὸν ὁρῶν ξυνιστάμενον πρὸς ἑκατέρους, τὸ μὲν εὐθύς, τὸ δὲ καὶ διανοούμενον."
+
+        target = ['Θουκυδίδης', 'Ἀθηναῖος', 'ξυνέγραψε', 'τὸν', 'πόλεμον', 'τῶν', 'Πελοποννησίων', 'καὶ', 'Ἀθηναίων', ',', 'ὡς', 'ἐπολέμησαν', 'πρὸς', 'ἀλλήλους', ',', 'ἀρξάμενος', 'εὐθὺς', 'καθισταμένου', 'καὶ', 'ἐλπίσας', 'μέγαν', 'τε', 'ἔσεσθαι', 'καὶ', 'ἀξιολογώτατον', 'τῶν', 'προγεγενημένων', ',', 'τεκμαιρόμενος', 'ὅτι', 'ἀκμάζοντές', 'τε', 'ᾖσαν', 'ἐς', 'αὐτὸν', 'ἀμφότεροι', 'παρασκευῇ', 'τῇ', 'πάσῃ', 'καὶ', 'τὸ', 'ἄλλο', 'Ἑλληνικὸν', 'ὁρῶν', 'ξυνιστάμενον', 'πρὸς', 'ἑκατέρους', ',', 'τὸ', 'μὲν', 'εὐθύς', ',', 'τὸ', 'δὲ', 'καὶ', 'διανοούμενον', '.']
+
+        result = word_tokenizer.tokenize(test)
+
+        self.assertEqual(result, target)
+
+        
     def test_latin_word_tokenizer(self):
         """Test Latin-specific word tokenizer."""
         word_tokenizer = WordTokenizer('latin')
@@ -213,7 +230,7 @@ class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
                   'vilja', 'þeira', '.']
         word_tokenizer = WordTokenizer('old_norse')
         result = word_tokenizer.tokenize(text)
-        print(result)
+        #print(result)
         self.assertTrue(result == target)
 
 if __name__ == '__main__':