Repository: piskvorky/gensim

Issue:
top-level __init__.py confuses cython; can we remove?
The top-level `__init__.py` seems to confuse cython – which apparently climbs the dir tree to find the 'package' directory. When my gensim clone is named something with a '-' (as is common, eg 'gensim-develop') cython fails with a message like "'gensim-develop.gensim.models.doc2vec_inner' is not a valid module name". My workaround has been to temporarily rename the directory for each compilation. 

But that leads to path cruft in the compiled C, and cython remains confused about the proper project-relative paths to things – which makes sharing definitions/functions (even between the sibling files word2vec_inner & doc2vec_inner) impossible. 

I was baffled by the behavior for a long time, and only after a post to cython-users did someone point me at the `__init__.py` as the issue. (See https://groups.google.com/d/msg/cython-users/d-3Clm2EkHs/G7CJish_c4kJ.) After locally removing `__init__.py`, I can skip the workaround and finally cimport a function from word2vec_inner to doc2vec_inner (rather than cut & paste). 

It looks like the file is a [workaround from ~4 years ago](https://github.com/piskvorky/gensim/commits/develop/__init__.py); the commit comment from @dedan says: "fix the module import when linking to the git root instead of module – for some application I need to link to the gensim folder which is also the root of the repository. This script helps python to find the actual sourcecode of the module and had to be changed because radim moved the source within the repo". 

That seems a pretty narrow need that may have expired. (Any thoughts, @dedan?) But I'd also be interested to know if there's another workaround others who've been preparing cython-compiled C files have used...



Discussion:
I agree, an unnecessary complication.

Let's try removing it for the next release and see whether anyone steps up to complain.


Resolving patch:
diff --git a/__init__.py b/__init__.py
deleted file mode 100644
--- a/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# script by dedan: helps him to symlink gensim
-import os
-dirname = __path__[0]		# Package's main folder
-__path__.insert(0, os.path.join(dirname, "gensim"))
diff --git a/gensim/models/doc2vec.py b/gensim/models/doc2vec.py
--- a/gensim/models/doc2vec.py
+++ b/gensim/models/doc2vec.py
@@ -36,6 +36,7 @@
 
 import logging
 import os
+import warnings
 
 try:
     from queue import Queue
@@ -56,8 +57,8 @@
 from six import string_types, integer_types
 
 try:
-    from gensim.models.doc2vec_inner import train_document_dbow, train_document_dm, train_document_dm_concat,\
-                                            FAST_VERSION
+    from gensim.models.doc2vec_inner import train_document_dbow, train_document_dm, train_document_dm_concat
+    from gensim.models.word2vec_inner import FAST_VERSION  # blas-adaptation shared from word2vec
 except:
     # failed... fall back to plain numpy (20-80x slower training than the above)
     FAST_VERSION = -1
@@ -242,6 +243,11 @@ class TaggedDocument(namedtuple('TaggedDocument','words tags')):
     def __str__(self):
         return '%s(%s, %s)' % (self.__class__.__name__, self.words, self.tags)
 
+# for compatibility
+class LabeledSentence(TaggedDocument):
+    def __init__(self, *args, **kwargs):
+        warnings.warn('LabeledSentence has been replaced by TaggedDocument', DeprecationWarning)
+
 
 class DocvecsArray(utils.SaveLoad):
     """
@@ -510,7 +516,7 @@ def __init__(self, documents=None, size=300, alpha=0.025, window=8, min_count=5,
         doc-vector training; default is 0 (faster training of doc-vectors only).
 
         """
-        Word2Vec.__init__(self, size=size, alpha=alpha, window=window, min_count=min_count,
+        super(Doc2Vec, self).__init__(size=size, alpha=alpha, window=window, min_count=min_count,
                           sample=sample, seed=seed, workers=workers, min_alpha=min_alpha,
                           sg=(1+dm) % 2, hs=hs, negative=negative, cbow_mean=dm_mean, 
                           null_word=dm_concat, **kwargs)
@@ -526,7 +532,7 @@ def __init__(self, documents=None, size=300, alpha=0.025, window=8, min_count=5,
             self.train(documents)
 
     def clear_sims(self):
-        Word2Vec.clear_sims(self)
+        super(Doc2Vec,self).clear_sims()
         self.docvecs.clear_sims()
 
     def reset_weights(self):
@@ -534,13 +540,13 @@ def reset_weights(self):
             # expand l1 size to match concatenated tags+words length
             self.layer1_size = (self.dm_tag_count + (2 * self.window)) * self.vector_size
             logger.info("using concatenative %d-dimensional layer1"% (self.layer1_size))
-        Word2Vec.reset_weights(self)
+        super(Doc2Vec,self).reset_weights()
         self.docvecs.reset_weights(self)
 
     def reset_from(self, other_model):
         """Reuse shareable structures from other_model."""
         self.docvecs.borrow_from(other_model.docvecs)
-        Word2Vec.reset_from(self, other_model)
+        super(Doc2Vec,self).reset_from(other_model)
 
     def _vocab_from(self, documents):
         document_no, vocab = -1, {}
diff --git a/gensim/models/word2vec.py b/gensim/models/word2vec.py
--- a/gensim/models/word2vec.py
+++ b/gensim/models/word2vec.py
@@ -112,7 +112,7 @@ def train_sentence_sg(model, sentence, alpha, work=None):
         for pos, word in enumerate(sentence):
             if word is None:
                 continue  # OOV word in the input sentence => skip
-            reduced_window = random.randint(model.window)  # `b` in the original word2vec code
+            reduced_window = model.random.randint(model.window)  # `b` in the original word2vec code
 
             # now go over all words from the (reduced) window, predicting each one in turn
             start = max(0, pos - model.window + reduced_window)
@@ -137,7 +137,7 @@ def train_sentence_cbow(model, sentence, alpha, work=None, neu1=None):
         for pos, word in enumerate(sentence):
             if word is None:
                 continue  # OOV word in the input sentence => skip
-            reduced_window = random.randint(model.window) # `b` in the original word2vec code
+            reduced_window = model.random.randint(model.window) # `b` in the original word2vec code
             start = max(0, pos - model.window + reduced_window)
             window_pos = enumerate(sentence[start : pos + model.window + 1 - reduced_window], start)
             word2_indices = [word2.index for pos2, word2 in window_pos if (word2 is not None and pos2 != pos)]
@@ -174,7 +174,7 @@ def train_sg_pair(model, predict_word, context_index, alpha, learn_vectors=True,
         # use this word (label = 1) + `negative` other random words not from this sentence (label = 0)
         word_indices = [predict_word.index]
         while len(word_indices) < model.negative + 1:
-            w = model.table[random.randint(model.table.shape[0])]
+            w = model.cum_table.searchsorted(model.random.randint(model.cum_table[-1]))
             if w != predict_word.index:
                 word_indices.append(w)
         l2b = model.syn1neg[word_indices]  # 2d matrix, k+1 x layer1_size
@@ -203,7 +203,7 @@ def train_cbow_pair(model, word, input_word_indices, l1, alpha, learn_vectors=Tr
         # use this word (label = 1) + `negative` other random words not from this sentence (label = 0)
         word_indices = [word.index]
         while len(word_indices) < model.negative + 1:
-            w = model.table[random.randint(model.table.shape[0])]
+            w = model.cum_table.searchsorted(model.random.randint(model.cum_table[-1]))
             if w != word.index:
                 word_indices.append(w)
         l2b = model.syn1neg[word_indices] # 2d matrix, k+1 x layer1_size
@@ -291,7 +291,7 @@ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5,
         self.vocab = {}  # mapping from a word (string) to a Vocab object
         self.index2word = []  # map from a word's matrix index (int) to word (string)
         self.sg = int(sg)
-        self.table = None # for negative sampling --> this needs a lot of RAM! consider setting back to None before saving
+        self.cum_table = None # for negative sampling
         self.vector_size = int(size)
         self.layer1_size = int(size)
         if size % 4 != 0:
@@ -299,6 +299,7 @@ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5,
         self.alpha = float(alpha)
         self.window = int(window)
         self.seed = seed
+        self.random = random.RandomState(seed)
         self.min_count = min_count
         self.sample = sample
         self.workers = workers
@@ -315,36 +316,29 @@ def __init__(self, sentences=None, size=100, alpha=0.025, window=5, min_count=5,
             self.build_vocab(sentences)
             self.train(sentences)
 
-    def make_table(self, table_size=100000000, power=0.75):
+
+    def make_cum_table(self, power=0.75, domain=2**31 - 1):
         """
-        Create a table using stored vocabulary word counts for drawing random words in the negative
-        sampling training routines.
+        Create a cumulative-distribution table using stored vocabulary word counts for
+        drawing random words in the negative-sampling training routines.
 
-        Called internally from `build_vocab()`.
+        To draw a word index, choose a random integer up to the maximum value in the
+        table (cum_table[-1]), then finding that integer's sorted insertion point
+        (as if by bisect_left or ndarray.searchsorted()). That insertion point is the
+        drawn index, coming up in proportion equal to the increment at that slot.
 
+        Called internally from 'build_vocab()'.
         """
-        logger.info("constructing a table with noise distribution from %i words" % len(self.vocab))
-        # table (= list of words) of noise distribution for negative sampling
         vocab_size = len(self.index2word)
-        self.table = zeros(table_size, dtype=uint32)
-
-        if not vocab_size:
-            logger.warning("empty vocabulary in word2vec, is this intended?")
-            return
-
+        self.cum_table = zeros(vocab_size, dtype=uint32)
         # compute sum of all power (Z in paper)
         train_words_pow = float(sum([self.vocab[word].count**power for word in self.vocab]))
-        # go through the whole table and fill it up with the word indices proportional to a word's count**power
-        widx = 0
-        # normalize count^0.75 by Z
-        d1 = self.vocab[self.index2word[widx]].count**power / train_words_pow
-        for tidx in xrange(table_size):
-            self.table[tidx] = widx
-            if 1.0 * tidx / table_size > d1:
-                widx += 1
-                d1 += self.vocab[self.index2word[widx]].count**power / train_words_pow
-            if widx >= vocab_size:
-                widx = vocab_size - 1
+        cumulative = 0.0
+        for word_index in range(vocab_size):
+            cumulative += self.vocab[self.index2word[word_index]].count**power / train_words_pow
+            self.cum_table[word_index] = round(cumulative * domain)
+        assert self.cum_table[-1] == domain
+
 
     def create_binary_tree(self):
         """
@@ -417,7 +411,7 @@ def build_vocab(self, sentences):
             self.create_binary_tree()
         if self.negative:
             # build the table for drawing random words (for negative sampling)
-            self.make_table()
+            self.make_cum_table()
         # precalculate downsampling thresholds
         self.precalc_sampling()
         self.reset_weights()
@@ -447,7 +441,7 @@ def reset_from(self, other_model):
         """
         self.vocab = other_model.vocab
         self.index2word = other_model.index2word
-        self.table = other_model.table
+        self.cum_table = other_model.cum_table
         self.reset_weights()
 
     def _prepare_items(self, items):
@@ -560,9 +554,9 @@ def reset_weights(self):
 
     def seeded_vector(self, seed_string):
         """Create one 'random' vector (but deterministic by seed_string)"""
-        # Note: Python's built in hash function can vary across versions of Python
-        random.seed(uint32(self.hashfxn(seed_string)))
-        return (random.rand(self.vector_size) - 0.5) / self.vector_size
+        # Note: built-in hash() may vary by Python version or even (in Py3.x) per launch
+        once = random.RandomState(uint32(self.hashfxn(seed_string)))
+        return (once.rand(self.vector_size) - 0.5) / self.vector_size
 
     def save_word2vec_format(self, fname, fvocab=None, binary=False):
         """
@@ -673,9 +667,8 @@ def intersect_word2vec_format(self, fname, binary=False):
             header = utils.to_unicode(fin.readline())
             vocab_size, vector_size = map(int, header.split())  # throws for invalid file format
             if not vector_size == self.vector_size:
-                logger.error("incompatible vector sizes")
-                # TOCONSIDER: maybe truncation/smaller vectors still useful enough to merge?
-                return # TODO raise ValueError()?
+                raise ValueError("incompatible vector size %d in file %s" % (vector_size, fname))
+                # TOCONSIDER: maybe mismatched vectors still useful enough to merge (truncating/padding)?
             if binary:
                 binary_len = dtype(REAL).itemsize * vector_size
                 for line_no in xrange(vocab_size):
@@ -994,16 +987,19 @@ def __str__(self):
 
     def save(self, *args, **kwargs):
         # don't bother storing the cached normalized vectors, recalculable table
-        kwargs['ignore'] = kwargs.get('ignore', ['syn0norm','table'])
+        kwargs['ignore'] = kwargs.get('ignore', ['syn0norm','table','cum_table'])
         super(Word2Vec, self).save(*args, **kwargs)
+
     save.__doc__ = utils.SaveLoad.save.__doc__
 
     
     @classmethod
     def load(cls, *args, **kwargs):
         model = super(Word2Vec, cls).load(*args, **kwargs)
-        if model.negative and not model.table:
-            model.make_table()  # rebuild table if missing
+        if hasattr(model,'table'):
+            delattr(model,'table')  # discard in favor of cum_table
+        if model.negative:
+            model.make_cum_table()  # rebuild cum_table from vocabulary
         return model
 
     


Test patch: