Repository: piskvorky/gensim

Issue:
dict.iteritems() usage breaks "keywords" function in python 3
Calling `gensim.summarization.keywords` in python3 results in `AttributeError: 'dict' object has no attribute 'iteritems'`.

Looks like there are offenders [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L35) and [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L114). I assume these should be fixed with six.iteritems?



Discussion:
Indeed! Thanks for reporting.

Can you put together a PR fix? And, ideally, also add a simple unit test for `summarization.keywords`, so we can catch such things more easily.

Sure, the iteritems fix should be easy enough, but to get the test working, I'm having trouble figuring out what to do with the `.decode("utf-8")` call [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L159).

It works fine with python 2, but in python3, I'm getting `AttributeError: 'str' object has no attribute 'decode'`. I'm not an expert with string encoding, so I'm not even sure what case `decode` is supposed to cover, but FWIW dropping the `decode` call doesn't change the results in python 2 for the an ascii example input I tried. (in python 2, calling `decode` with some unicode input failed with `UnicodeEncodeError: 'ascii' codec can't encode character u'\u201c' in position 1: ordinal not in range(128)`. So maybe the purpose of the call is to ensure ascii-only input?)

Any suggestions? Or should I open a separate issue?

We want to convert all text inputs to **unicode** -- best use `gensim.utils.to_unicode`. It accepts both utf8 and unicode, so it's safe for whatever the user throws at us.

Similarly, on output, we convert to utf8 using `gensim.utils.to_utf8`. Again, it accepts both utf8 and unicode as input, so it's safer.

We certainly don't want to enforce ASCII inputs :)


Resolving patch:
diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py
--- a/gensim/summarization/keywords.py
+++ b/gensim/summarization/keywords.py
@@ -8,9 +8,11 @@
 from gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word
 from gensim.summarization.commons import build_graph as _build_graph
 from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes
+from gensim.utils import to_unicode
 from itertools import combinations as _combinations
 from six.moves.queue import Queue as _Queue
 from six.moves import xrange
+from six import iteritems
 
 
 WINDOW_SIZE = 2
@@ -32,7 +34,7 @@ def _get_words_for_graph(tokens):
         raise ValueError("Can't use both include and exclude filters, should use only one")
 
     result = []
-    for word, unit in tokens.iteritems():
+    for word, unit in iteritems(tokens):
         if exclude_filters and unit.tag in exclude_filters:
             continue
         if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:
@@ -111,7 +113,7 @@ def _extract_tokens(lemmas, scores, ratio, words):
 
 def _lemmas_to_words(tokens):
     lemma_to_word = {}
-    for word, unit in tokens.iteritems():
+    for word, unit in iteritems(tokens):
         lemma = unit.token
         if lemma in lemma_to_word:
             lemma_to_word[lemma].append(word)
@@ -156,7 +158,7 @@ def _get_combined_keywords(_keywords, split_text):
                 result.append(word)   # appends last word if keyword and doesn't iterate
             for j in xrange(i + 1, len_text):
                 other_word = _strip_word(split_text[j])
-                if other_word in _keywords and other_word == split_text[j].decode("utf-8"):
+                if other_word in _keywords and other_word == split_text[j]:
                     combined_word.append(other_word)
                 else:
                     for keyword in combined_word:
@@ -191,6 +193,7 @@ def _format_results(_keywords, combined_keywords, split, scores):
 
 def keywords(text, ratio=0.2, words=None, split=False, scores=False):
     # Gets a dict of word -> lemma
+    text = to_unicode(text)
     tokens = _clean_text_by_word(text)
     split_text = list(_tokenize_by_word(text))
 


Test patch:
diff --git a/gensim/test/test_summarization.py b/gensim/test/test_summarization.py
--- a/gensim/test/test_summarization.py
+++ b/gensim/test/test_summarization.py
@@ -17,7 +17,7 @@
 
 from gensim import utils
 from gensim.corpora import Dictionary
-from gensim.summarization import summarize, summarize_corpus
+from gensim.summarization import summarize, summarize_corpus, keywords
 
 
 class TestSummarizationTest(unittest.TestCase):
@@ -128,3 +128,18 @@ def test_corpus_summarization_ratio(self):
             expected_summary_length = int(len(corpus) * ratio)
 
             self.assertEqual(len(selected_docs), expected_summary_length)
+
+    def test_keywords_runs(self):
+        pre_path = os.path.join(os.path.dirname(__file__), 'test_data')
+
+        with utils.smart_open(os.path.join(pre_path, "mihalcea_tarau.txt")) as f:
+            text = f.read()
+
+        kwds = keywords(text)
+        self.assertTrue(len(kwds.splitlines()))
+
+        kwds_u = keywords(utils.to_unicode(text))
+        self.assertTrue(len(kwds_u.splitlines()))
+
+        kwds_lst = keywords(text, split=True)
+        self.assertTrue(len(kwds_lst))