Repository: cltk/cltk

Issue:
Greek not among word tokenizers
Hi! I attempted to use the Greek word tokenizer for a really quick search of a plain text file, as follows:

<pre>
from cltk.tokenize.word import WordTokenizer

word_tokenizer = WordTokenizer('greek')

for word in word_tokenizer.tokenize('files/plain_text/1-23_1.1-42.txt'):
	if word.endswith('εαι'):
		print(word)
</pre>

When I tried to call the script, I got the following error:

`AssertionError: Specific tokenizer not available for 'greek'. Only available for: '['arabic', 'latin', 'french', 'old_norse']'.`

Thanks!


Discussion:

Resolving patch:
diff --git a/cltk/stop/classical_hindi/__init__.py b/cltk/stop/classical_hindi/__init__.py
new file mode 100644
diff --git a/cltk/stop/classical_hindi/stops.py b/cltk/stop/classical_hindi/stops.py
new file mode 100644
--- /dev/null
+++ b/cltk/stop/classical_hindi/stops.py
@@ -0,0 +1,108 @@
+""" Classical Hindi Stopwords
+This list is composed from 100 most frequently occuring words in classical_hindi corpus <https://github.com/cltk/hindi_text_ltrc> in CLTK.
+source code : <https://gist.github.com/inishchith/ad4bc0da200110de638f5408c64bb14c>
+"""
+__author__ = 'Nishchith Shetty <inishchith[at]gmail[.]com>'
+
+STOP_LIST = ["हें", # yes
+              "है", # is
+              "हैं", # there
+              "हि", # this
+              "ही", # only
+              "हो", # may
+              "हे", #
+              "से", # from
+              "अत", # so
+              "के",  # of
+              "रहे", # are
+              "का", # of
+              "की", # of
+              "कि", # that
+              "तो", # so
+              "ने", # has
+              "एक", # one
+              "नहीं", # no
+              "पे", # on
+              "में", # in
+              "वाले", # ones
+              "सकते", # can
+              "वह", # he
+              "वे" , # they
+              "कई", # many
+              "होती", # was
+              "आप", # you
+              "यह", # this
+              "और", # and
+              "एवं", # and
+              "को", # to
+              "मे", # in
+              "दो", # two
+              "थे", # were
+              "यदि", # if
+              "उनके", # their
+              "थी" , # was
+              "पर", # on
+              "इस", # this
+              "साथ", # with
+              "लिए", # for
+              "जो", # that
+              "होता", # happen
+              "या", # or
+              "लिये", # for
+              "द्वारा", # by
+              "हुई", # was done
+              "जब", # when
+              "होते", # were
+              "व", # and
+              "न", # not
+              "उनकी", # their
+              "आदि", # more
+              "सकता", # can
+              "उनका", # their
+              "इतयादि", # e.t.c
+              "इतना", # this much
+              "जिस", # which
+              "उस", # that
+              "कैसे",# how
+              "हूँ", # am
+              "ना", # no
+              "कहि", # say
+              "सम", # even
+              "र्", # the
+              "कहँ", # where
+              "बस", # enough
+              "अपना", # ours
+              "यही", # this  only
+              "कहीं", # somewhere
+              "हाँ", # yes
+              "मैंने", # i
+              "जहँ", # where
+              "सब", # all
+              "यह", # this
+              "था", # was
+              "तुम", # you
+              "ये", # these
+              "जे", # which
+              "भी", # as well / also
+              "हम", # we
+              "अब", # now
+              "ऐसे", # such
+              "वहाँ", # there
+              "क्या", # what
+              "ओर", # and
+              "इसी", # this
+              "सके", # could
+              "कभी", # sometimes
+              "हर", # every
+              "मेरी", # my
+              "कम", # less
+              "सा",
+              "उन्हें", # them
+              "मेरे", # my
+              "उन", # those
+              "कुछ", # some
+              "इन", # these
+              "ऐसा", # like these
+              "जहा", # where
+              "तीन", # three
+]


Test patch:
diff --git a/cltk/tests/test_corpus.py b/cltk/tests/test_corpus.py
--- a/cltk/tests/test_corpus.py
+++ b/cltk/tests/test_corpus.py
@@ -729,7 +729,7 @@ def test_in_coordinated_range(self):
 
     def test_is_indiclang_char(self):
         self.assertTrue(is_indiclang_char('क', 'hi'))
-        
+
     def test_swadesh_greek(self):
         swadesh = Swadesh('gr')
         first_word = 'ἐγώ'