Repository: piskvorky/gensim

Issue:
Error while summarizing text
Hi,

I've received the following error when trying to summarize the body of this news article:

https://www.theguardian.com/media/2016/jun/19/sun-times-brexit-in-out-shake-it-all-about

The error follows:

  File "/home/apps/comment_parser/venv/local/lib/python2.7/site-packages/gensim/summarization/summarizer.py", line 202, in summarize
    most_important_docs = summarize_corpus(corpus, ratio=ratio if word_count is None else 1)
  File "/home/apps/comment_parser/venv/local/lib/python2.7/site-packages/gensim/summarization/summarizer.py", line 161, in summarize_corpus
    pagerank_scores = _pagerank(graph)
  File "/home/apps/comment_parser/venv/local/lib/python2.7/site-packages/gensim/summarization/pagerank_weighted.py", line 24, in pagerank_weighted
    vals, vecs = eigs(pagerank_matrix.T, k=1)  # TODO raise an error if matrix has complex eigenvectors?
  File "/usr/lib/python2.7/dist-packages/scipy/sparse/linalg/eigen/arpack/arpack.py", line 1271, in eigs
    ncv, v0, maxiter, which, tol)
  File "/usr/lib/python2.7/dist-packages/scipy/sparse/linalg/eigen/arpack/arpack.py", line 685, in __init__
    raise ValueError("k must be less than ndim(A)-1, k=%d" % k)
ValueError: k must be less than ndim(A)-1, k=1

Regards,



Discussion:
I am also affected by this bug...

@madewild could you post a code snippet to reproduce?

Unfortunately the text fed to the summarizer is confidential, but my guess is that the error was triggered by an unusually high repetition of some sentences... I also notice now that the error raised was not exactly the same:
ValueError: k must be less than **rank**(A)-1, k=1

@tmylk The reason for this failure looks like the number of nodes in the graph that is used to calculate the pagerank of the corpus graph, after removing unreachable nodes the graph is left with only 2 nodes and hence it builds a matrix of shape 2 \* 2 (for which scipy.sparse.linalg.eigs() will fail for k=1). We should probably raise an error if number of nodes (after removing unreachable nodes) goes below 3.

@MridulS could you submit a pr for this?

@tmylk What kind of error should I raise?

The error should say "Please add more sentences to the text. The number of reachable nodes is below 3"


Resolving patch:
diff --git a/gensim/summarization/summarizer.py b/gensim/summarization/summarizer.py
--- a/gensim/summarization/summarizer.py
+++ b/gensim/summarization/summarizer.py
@@ -158,6 +158,11 @@ def summarize_corpus(corpus, ratio=0.2):
     _set_graph_edge_weights(graph)
     _remove_unreachable_nodes(graph)
 
+    # Cannot calculate eigenvectors if number of unique words in text < 3. Warns user to add more text. The function ends.
+    if len(graph.nodes()) < 3:
+        logger.warning("Please add more sentences to the text. The number of reachable nodes is below 3")
+        return
+
     pagerank_scores = _pagerank(graph)
 
     hashable_corpus.sort(key=lambda doc: pagerank_scores.get(doc, 0), reverse=True)


Test patch:
diff --git a/gensim/test/test_data/testlowdistinctwords.txt b/gensim/test/test_data/testlowdistinctwords.txt
new file mode 100644
--- /dev/null
+++ b/gensim/test/test_data/testlowdistinctwords.txt
@@ -0,0 +1,10 @@
+here here.
+there there. 
+here here.
+there there.
+here here.
+there there.
+here here.
+there there.
+here here.
+there there.
\ No newline at end of file
diff --git a/gensim/test/test_summarization.py b/gensim/test/test_summarization.py
--- a/gensim/test/test_summarization.py
+++ b/gensim/test/test_summarization.py
@@ -160,6 +160,20 @@ def test_keywords_runs(self):
         kwds_lst = keywords(text, split=True)
         self.assertTrue(len(kwds_lst))
 
+    def test_low_distinct_words_corpus_summarization_is_none(self):
+        pre_path = os.path.join(os.path.dirname(__file__), 'test_data')
+
+        with utils.smart_open(os.path.join(pre_path, "testlowdistinctwords.txt"), mode="r") as f:
+            text = f.read()
+
+        # Generate the corpus.
+        sentences = text.split("\n")
+        tokens = [sentence.split() for sentence in sentences]
+        dictionary = Dictionary(tokens)
+        corpus = [dictionary.doc2bow(sentence_tokens) for sentence_tokens in tokens]
+
+        self.assertTrue(summarize_corpus(corpus) is None)
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()
diff --git a/gensim/test/test_wikicorpus.py b/gensim/test/test_wikicorpus.py
--- a/gensim/test/test_wikicorpus.py
+++ b/gensim/test/test_wikicorpus.py
@@ -27,10 +27,11 @@ class TestWikiCorpus(unittest.TestCase):
     def setUp(self):
         wc = WikiCorpus(datapath(FILENAME))
 
-
     def test_get_texts_returns_generator_of_lists(self):
+        
         if sys.version_info < (2, 7, 0):
             return
+        
         wc = WikiCorpus(datapath(FILENAME))
         l = wc.get_texts()
         self.assertEqual(type(l), types.GeneratorType)