Repository: piskvorky/gensim

Issue:
Doc2Vec: mixing plain int and string tags will give unexpected results
For typical Doc2Vec use with a large corpus and only a single (ID) tag per document, the best approach is to use contiguous plain ints starting from 0. (It avoids creating a string tag->index dict.)

Docs can, however, have more than one tag, and it's quite natural to think such other tags will be symbolic, just as a category-name. However, mixing plain-int and string tags in the same corpus will lead to confusing results: the ints are taken as literal indexes into the array of docvecs-in-trainig, but any string is assigned the next-available-int when it is first encounterd. Unless the user has been careful not to use that same int as a plain int ID, the int and string will indvertently share the same vector. 

Using _only_ ints or strings will work well in the current code, but the scan can probably be adapted to avoid this problem. (First idea: strings are given temporary provisional indexes upon first discovery: perhaps, negative indexes. Only at the end of the full corpus scan are all string tags – those in `model.docvecs.doctags` and `model.docvecs.index2doctag` given real final positive int indexes, at the end of all other indexes. In the case where only string tags are used, their assigned indexes will wind up as the same starting-from-zero ints as before.)



Discussion:

Resolving patch:
diff --git a/gensim/models/doc2vec.py b/gensim/models/doc2vec.py
--- a/gensim/models/doc2vec.py
+++ b/gensim/models/doc2vec.py
@@ -278,21 +278,22 @@ class DocvecsArray(utils.SaveLoad):
     """
     def __init__(self, mapfile_path=None):
         self.doctags = {}  # string -> Doctag (only filled if necessary)
-        self.index2doctag = []  # int index -> String (only filled if necessary)
+        self.max_rawint = -1  # highest rawint-indexed doctag
+        self.offset2doctag = []  # int offset-past-(max_rawint+1) -> String (only filled if necessary)
         self.count = 0
         self.mapfile_path = mapfile_path
 
     def note_doctag(self, key, document_no, document_length):
         """Note a document tag during initial corpus scan, for structure sizing."""
         if isinstance(key, int):
-            self.count = max(self.count, key+1)
+            self.max_rawint = max(self.max_rawint, key)
         else:
             if key in self.doctags:
                 self.doctags[key] = self.doctags[key].repeat(document_length)
             else:
-                self.doctags[key] = Doctag(len(self.index2doctag), document_length, 1)
-                self.index2doctag.append(key)
-                self.count = max(self.count, len(self.index2doctag))
+                self.doctags[key] = Doctag(len(self.offset2doctag), document_length, 1)
+                self.offset2doctag.append(key)
+        self.count = self.max_rawint + 1 + len(self.offset2doctag)
 
     def indexed_doctags(self, doctag_tokens):
         """Return indexes and backing-arrays used in training examples."""
@@ -309,12 +310,18 @@ def _int_index(self, index, missing=None):
         if isinstance(index, int):
             return index
         else:
-            return self.doctags[index].index if index in self.doctags else missing
+            return self.max_rawint + 1 + self.doctags[index].offset if index in self.doctags else missing
 
     def _key_index(self, i_index, missing=None):
         """Return string index for given int index, if available"""
-        if i_index < len(self.index2doctag):
-            return self.index2doctag[i_index]
+        warnings.warn("use DocvecsArray.index_to_doctag", DeprecationWarning)
+        return self.index_to_doctag(i_index)
+
+    def index_to_doctag(self, i_index):
+        """Return string key for given i_index, if available. Otherwise return raw int doctag (same int)."""
+        candidate_offset = self.max_rawint - i_index - 1
+        if 0 <= candidate_offset < len(self.offset2doctag):
+            return self.offset2doctag[candidate_offset]
         else:
             return i_index
 
@@ -345,14 +352,14 @@ def __contains__(self, index):
     def borrow_from(self, other_docvecs):
         self.count = other_docvecs.count
         self.doctags = other_docvecs.doctags
-        self.index2doctag = other_docvecs.index2doctag
+        self.offset2doctag = other_docvecs.offset2doctag
 
     def clear_sims(self):
         self.doctag_syn0norm = None
 
     def estimated_lookup_memory(self):
         """Estimated memory for tag lookup; 0 if using pure int tags."""
-        return 60 * len(self.index2doctag) + 140 * len(self.doctags)
+        return 60 * len(self.offset2doctag) + 140 * len(self.doctags)
 
     def reset_weights(self, model):
         length = max(len(self.doctags), self.count)
@@ -368,7 +375,7 @@ def reset_weights(self, model):
 
         for i in xrange(length):
             # construct deterministic seed from index AND model seed
-            seed = "%d %s" % (model.seed, self.index2doctag[i] if len(self.index2doctag) > 0 else str(i))
+            seed = "%d %s" % (model.seed, self.index_to_doctag(i))
             self.doctag_syn0[i] = model.seeded_vector(seed)
 
     def init_sims(self, replace=False):
@@ -447,7 +454,7 @@ def most_similar(self, positive=[], negative=[], topn=10, clip_start=0, clip_end
             return dists
         best = matutils.argsort(dists, topn=topn + len(all_docs), reverse=True)
         # ignore (don't return) docs from the input
-        result = [(self._key_index(sim), float(dists[sim])) for sim in best if sim not in all_docs]
+        result = [(self.index_to_doctag(sim), float(dists[sim])) for sim in best if sim not in all_docs]
         return result[:topn]
 
     def doesnt_match(self, docs):
@@ -487,11 +494,16 @@ def n_similarity(self, ds1, ds2):
         return dot(matutils.unitvec(array(v1).mean(axis=0)), matutils.unitvec(array(v2).mean(axis=0)))
 
 
-class Doctag(namedtuple('Doctag', 'index, word_count, doc_count')):
+class Doctag(namedtuple('Doctag', 'offset, word_count, doc_count')):
     """A string document tag discovered during the initial vocabulary
     scan. (The document-vector equivalent of a Vocab object.)
 
     Will not be used if all presented document tags are ints.
+
+    The offset is only the true index into the doctags_syn0/doctags_syn0_lockf
+    if-and-only-if no raw-int tags were used. If any raw-int tags were used,
+    string Doctag vectors begin at index (max_rawint + 1), so the true index is
+    (rawint_index + 1 + offset). See also DocvecsArray.index_to_doctag().
     """
     __slots__ = ()
 


Test patch:
diff --git a/gensim/test/test_doc2vec.py b/gensim/test/test_doc2vec.py
--- a/gensim/test/test_doc2vec.py
+++ b/gensim/test/test_doc2vec.py
@@ -41,7 +41,7 @@ def __iter__(self):
 
 list_corpus = list(DocsLeeCorpus())
 
-sentences = [
+raw_sentences = [
         ['human', 'interface', 'computer'],
         ['survey', 'user', 'computer', 'system', 'response', 'time'],
         ['eps', 'user', 'interface', 'system'],
@@ -53,8 +53,7 @@ def __iter__(self):
         ['graph', 'minors', 'survey']
     ]
 
-sentences = [doc2vec.TaggedDocument(words, [i]) for i, words in enumerate(sentences)]
-
+sentences = [doc2vec.TaggedDocument(words, [i]) for i, words in enumerate(raw_sentences)]
 
 def testfile():
     # temporary data will be stored to this file
@@ -101,7 +100,8 @@ def test_string_doctags(self):
         self.assertEqual(model.docvecs[0].shape, (300,))
         self.assertEqual(model.docvecs['_*0'].shape, (300,))
         self.assertTrue(all(model.docvecs['_*0'] == model.docvecs[0]))
-        self.assertTrue(max(d.index for d in model.docvecs.doctags.values()) < len(model.docvecs.doctag_syn0))
+        self.assertTrue(max(d.offset for d in model.docvecs.doctags.values()) < len(model.docvecs.doctags))
+        self.assertTrue(max(model.docvecs._int_index(str_key) for str_key in model.docvecs.doctags.keys()) < len(model.docvecs.doctag_syn0))
 
     def test_empty_errors(self):
         # no input => "RuntimeError: you must first build vocabulary before training the model"
@@ -235,6 +235,16 @@ def test_deterministic_dmc(self):
                                  seed=42, workers=1)
         self.models_equal(model, model2)
 
+    def test_mixed_tag_types(self):
+        """Ensure alternating int/string tags don't share indexes in doctag_syn0"""
+        mixed_tag_corpus = [doc2vec.TaggedDocument(words, [i, words[0]]) for i, words in enumerate(raw_sentences)]
+        model = doc2vec.Doc2Vec()
+        model.build_vocab(mixed_tag_corpus)
+        expected_length = len(sentences) + len(model.docvecs.doctags)  # 9 sentences, 7 unique first tokens
+        print(model.docvecs.doctags)
+        print(model.docvecs.count)
+        self.assertEquals(len(model.docvecs.doctag_syn0), expected_length)
+
     def models_equal(self, model, model2):
         # check words/hidden-weights
         self.assertEqual(len(model.vocab), len(model2.vocab))
@@ -245,7 +255,7 @@ def models_equal(self, model, model2):
             self.assertTrue(np.allclose(model.syn1neg, model2.syn1neg))
         # check docvecs
         self.assertEqual(len(model.docvecs.doctags), len(model2.docvecs.doctags))
-        self.assertEqual(len(model.docvecs.index2doctag), len(model2.docvecs.index2doctag))
+        self.assertEqual(len(model.docvecs.offset2doctag), len(model2.docvecs.offset2doctag))
         self.assertTrue(np.allclose(model.docvecs.doctag_syn0, model2.docvecs.doctag_syn0))
 
 #endclass TestDoc2VecModel