Repository: piskvorky/gensim

Issue:
SparseTermSimilarityMatrix - TypeError: 'numpy.float32' object is not iterable
I am using gensim 3.7.3 and python3.6.

I am following the exact example of SoftCosineSimilarity at https://radimrehurek.com/gensim/similarities/docsim.html
but with my own dataset and embeddings trained on Fasttext.
Dictionary and WordEmbeddingSimilarityIndex are executed properly but then I get an error when trying SparseTermSimilarityMatrix. I found a similar issue, that was solved in the pull below, but I still seem to get this error. However, I tried the exact same code with Word2Vec and the gensim imported common_texts and it worked. Why it doesnt work in my case, is it related to FastText?

https://github.com/RaRe-Technologies/gensim/pull/2356

My code:
```
from gensim.models import FastText
from gensim.corpora import Dictionary
from gensim.models import WordEmbeddingSimilarityIndex
from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix

model = FastText.load('fasttext_vector_100')
# this line works
model.wv.most_similar(positive=['test'], topn=2)
termsim_index = WordEmbeddingSimilarityIndex(model.wv)
# texts is similar to common_texts, list of lists of strings
dictionary = Dictionary(texts)
bow_corpus = [dictionary.doc2bow(document) for document in texts]
# it fails here
similarity_matrix = SparseTermSimilarityMatrix(termsim_index, dictionary)
```


```
TypeError                                 Traceback (most recent call last)
<ipython-input-129-c33cf3beaa3e> in <module>()
----> 1 similarity_matrix = SparseTermSimilarityMatrix(termsim_index, dictionary)  # construct similarity matrix

~\AppData\Local\Continuum\anaconda3\lib\site-packages\gensim\similarities\termsim.py in __init__(self, source, dictionary, tfidf, symmetric, positive_definite, nonzero_limit, dtype)
    232             most_similar = [
    233                 (dictionary.token2id[term], similarity)
--> 234                 for term, similarity in index.most_similar(t1, num_rows)
    235                 if term in dictionary.token2id]
    236 

~\AppData\Local\Continuum\anaconda3\lib\site-packages\gensim\similarities\termsim.py in <listcomp>(.0)
    231             num_rows = nonzero_limit - num_nonzero
    232             most_similar = [
--> 233                 (dictionary.token2id[term], similarity)
    234                 for term, similarity in index.most_similar(t1, num_rows)
    235                 if term in dictionary.token2id]

~\AppData\Local\Continuum\anaconda3\lib\site-packages\gensim\models\keyedvectors.py in most_similar(self, t1, topn)
   1418         else:
   1419             most_similar = self.keyedvectors.most_similar(positive=[t1], topn=topn, **self.kwargs)
-> 1420             for t2, similarity in most_similar:
   1421                 if similarity > self.threshold:
   1422                     yield (t2, similarity**self.exponent)

TypeError: 'numpy.float32' object is not iterable

```


Discussion:
@Witiko can you please have a look?
Hi there, 

I came across the same issue a few days ago.
I was about to open an issue as well to offer a fix I found.

The issue lies in line 248 of gensim/termsim.py
There is a test intended to ensure no more than nonzero_limit elements are inserted. The test is off and allows this limit to be broken, which makes the loop break at the next iteration.

The fix is simply to replace the "<=" test with a "<".

Hope this helps you guys!

Resolving patch:
diff --git a/gensim/models/keyedvectors.py b/gensim/models/keyedvectors.py
--- a/gensim/models/keyedvectors.py
+++ b/gensim/models/keyedvectors.py
@@ -162,6 +162,7 @@
 
 from itertools import chain
 import logging
+from numbers import Integral
 
 try:
     from queue import Queue, Empty
@@ -519,7 +520,7 @@ def most_similar(self, positive=None, negative=None, topn=10, restrict_vocab=Non
             one-dimensional numpy array with the size of the vocabulary.
 
         """
-        if isinstance(topn, int) and topn < 1:
+        if isinstance(topn, Integral) and topn < 1:
             return []
 
         if positive is None:
@@ -807,7 +808,7 @@ def most_similar_cosmul(self, positive=None, negative=None, topn=10):
             one-dimensional numpy array with the size of the vocabulary.
 
         """
-        if isinstance(topn, int) and topn < 1:
+        if isinstance(topn, Integral) and topn < 1:
             return []
 
         if positive is None:
@@ -1678,7 +1679,7 @@ def most_similar(self, positive=None, negative=None, topn=10, clip_start=0, clip
             Sequence of (doctag/index, similarity).
 
         """
-        if isinstance(topn, int) and topn < 1:
+        if isinstance(topn, Integral) and topn < 1:
             return []
 
         if positive is None:
diff --git a/gensim/models/poincare.py b/gensim/models/poincare.py
--- a/gensim/models/poincare.py
+++ b/gensim/models/poincare.py
@@ -44,6 +44,7 @@
 
 import csv
 import logging
+from numbers import Integral
 import sys
 import time
 
@@ -1189,7 +1190,8 @@ def most_similar(self, node_or_vector, topn=10, restrict_vocab=None):
         node_or_vector : {str, int, numpy.array}
             node key or vector for which similar nodes are to be found.
         topn : int or None, optional
-            number of similar nodes to return, if `None`, returns all.
+            Number of top-N similar nodes to return, when `topn` is int. When `topn` is None,
+            then distance for all nodes are returned.
         restrict_vocab : int or None, optional
             Optional integer which limits the range of vectors which are searched for most-similar values.
             For example, restrict_vocab=10000 would only check the first 10000 node vectors in the vocabulary order.
@@ -1197,8 +1199,10 @@ def most_similar(self, node_or_vector, topn=10, restrict_vocab=None):
 
         Returns
         --------
-        list of (str, float)
-            List of tuples containing (node, distance) pairs in increasing order of distance.
+        list of (str, float) or numpy.array
+            When `topn` is int, a sequence of (node, distance) is returned in increasing order of distance.
+            When `topn` is None, then similarities for all words are returned as a one-dimensional numpy array with the
+            size of the vocabulary.
 
         Examples
         --------
@@ -1216,6 +1220,9 @@ def most_similar(self, node_or_vector, topn=10, restrict_vocab=None):
             [(u'kangaroo.n.01', 0.0), (u'marsupial.n.01', 0.26524229460827725)]
 
         """
+        if isinstance(topn, Integral) and topn < 1:
+            return []
+
         if not restrict_vocab:
             all_distances = self.distances(node_or_vector)
         else:
diff --git a/gensim/similarities/termsim.py b/gensim/similarities/termsim.py
--- a/gensim/similarities/termsim.py
+++ b/gensim/similarities/termsim.py
@@ -209,7 +209,7 @@ def __init__(self, source, dictionary=None, tfidf=None, symmetric=True, positive
                     tfidf.idfs.items(),
                     key=lambda x: (lambda term_index, term_idf: (term_idf, -term_index))(*x), reverse=True)]
 
-        column_nonzero = np.array([1] * matrix_order, dtype=_shortest_uint_dtype(nonzero_limit))
+        column_nonzero = np.array([0] * matrix_order, dtype=_shortest_uint_dtype(nonzero_limit))
         column_sum = np.zeros(matrix_order, dtype=dtype)
         matrix = sparse.identity(matrix_order, dtype=dtype, format="dok")
 
@@ -227,12 +227,13 @@ def __init__(self, source, dictionary=None, tfidf=None, symmetric=True, positive
                         0.0, 1.0))
 
             t1 = dictionary[t1_index]
-            num_nonzero = column_nonzero[t1_index] - 1
+            num_nonzero = column_nonzero[t1_index]
             num_rows = nonzero_limit - num_nonzero
             most_similar = [
                 (dictionary.token2id[term], similarity)
-                for term, similarity in index.most_similar(t1, num_rows)
-                if term in dictionary.token2id]
+                for term, similarity in index.most_similar(t1, topn=num_rows)
+                if term in dictionary.token2id
+            ] if num_rows > 0 else []
 
             if tfidf is None:
                 rows = sorted(most_similar)
@@ -245,7 +246,7 @@ def __init__(self, source, dictionary=None, tfidf=None, symmetric=True, positive
                 if positive_definite and column_sum[t1_index] + abs(similarity) >= 1.0:
                     break
                 if symmetric:
-                    if column_nonzero[t2_index] <= nonzero_limit \
+                    if column_nonzero[t2_index] < nonzero_limit \
                             and (not positive_definite or column_sum[t2_index] + abs(similarity) < 1.0) \
                             and not (t1_index, t2_index) in matrix:
                         matrix[t1_index, t2_index] = similarity


Test patch:
diff --git a/gensim/test/test_keyedvectors.py b/gensim/test/test_keyedvectors.py
--- a/gensim/test/test_keyedvectors.py
+++ b/gensim/test/test_keyedvectors.py
@@ -109,6 +109,9 @@ def test_most_similar_topn(self):
         predicted = self.vectors.most_similar('war', topn=0)
         self.assertEqual(len(predicted), 0)
 
+        predicted = self.vectors.most_similar('war', topn=np.uint8(0))
+        self.assertEqual(len(predicted), 0)
+
     def test_relative_cosine_similarity(self):
         """Test relative_cosine_similarity returns expected results with an input of a word pair and topn"""
         wordnet_syn = [