Repository: piskvorky/gensim

Issue:
Make add_word part of keyedvectors.py interface
Hi, I think it would be very useful to define `add_word` inside the class instead of inside the `load` function in [keyedvectors](https://github.com/RaRe-Technologies/gensim/blob/7127eeef8304852da589465b338e2bba14e881f1/gensim/models/keyedvectors.py).

Thanks ✌🏻


Discussion:
Thanks for report @talbaumel, I saw some message in mailing list from users, who want to modify `KeyedVectors`, for me - sounds like a good idea, I'm +1 for this
Can I take this?
@MritunjayMohitesh yes
@talbaumel isn't it already done?
It's defiened under `load_word2vec_format `, that means it's not visible to users...
@talbaumel  please check this link out-
https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py

I found this in the develop branch. That's why I think it's already been done. I think I am not getting you.Could you please elaborate?
@MritunjayMohitesh 
The `add_word` method that should be publicly exposed is in [utils_any2vec.py#L182](https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/utils_any2vec.py#L182) in the development version of the code.
Thanks @englhardt 
As far as I understand, defining `add_word `outside the `load_word2vec_format` will require two additional parameters to be passed . Besides,I don't see this function being used anywhere else. So, I think defining outside would not be very much efficient. Please correct me if I'm still getting it wrong.
 I wanted to add a <PAD> and <EOS> tokens to pre-trained keyedvectors object.
It requiered some hacking since the function was not visible, I tought it might be a common scenario 🤷🏻‍♂️
@menshikh-iv Greetings, does anyone work on this? If not, I could prepare a PR.

I think it'd be better **not** to define current `add_word` function (which is defined inside of `load_word2vec_format`) outside, but write `BaseKeyedVectors.add_word` method (or another method name) from scratch, because it does not need some logic (e.g. with `counts` dict [here](https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/utils_any2vec.py#L190)).


@persiyanov hello, I see no PRs related with this change, feel free to contribute :+1: 

Resolving patch:
diff --git a/gensim/models/keyedvectors.py b/gensim/models/keyedvectors.py
--- a/gensim/models/keyedvectors.py
+++ b/gensim/models/keyedvectors.py
@@ -72,8 +72,8 @@
 except ImportError:
     PYEMD_EXT = False
 
-from numpy import dot, zeros, float32 as REAL, empty, memmap as np_memmap, \
-    double, array, vstack, sqrt, newaxis, integer, \
+from numpy import dot, float32 as REAL, empty, memmap as np_memmap, \
+    double, array, zeros, vstack, sqrt, newaxis, integer, \
     ndarray, sum as np_sum, prod, argmax, divide as np_divide
 import numpy as np
 from gensim import utils, matutils  # utility fnc for pickling, common scipy operations etc
@@ -109,7 +109,7 @@ def __str__(self):
 class BaseKeyedVectors(utils.SaveLoad):
 
     def __init__(self, vector_size):
-        self.vectors = []
+        self.vectors = zeros((0, vector_size))
         self.vocab = {}
         self.vector_size = vector_size
         self.index2entity = []
@@ -154,6 +154,65 @@ def get_vector(self, entity):
         else:
             raise KeyError("'%s' not in vocabulary" % entity)
 
+    def add(self, entities, weights, replace=False):
+        """Add entities and theirs vectors in a manual way.
+        If some entity is already in the vocabulary, old vector is keeped unless `replace` flag is True.
+
+        Parameters
+        ----------
+        entities : list of str
+            Entities specified by string tags.
+        weights: {list of numpy.ndarray, numpy.ndarray}
+            List of 1D np.array vectors or 2D np.array of vectors.
+        replace: bool, optional
+            Flag indicating whether to replace vectors for entities which are already in the vocabulary,
+            if True - replace vectors, otherwise - keep old vectors.
+
+        """
+        if isinstance(entities, string_types):
+            entities = [entities]
+            weights = np.array(weights).reshape(1, -1)
+        elif isinstance(weights, list):
+            weights = np.array(weights)
+
+        in_vocab_mask = np.zeros(len(entities), dtype=np.bool)
+        for idx, entity in enumerate(entities):
+            if entity in self.vocab:
+                in_vocab_mask[idx] = True
+
+        # add new entities to the vocab
+        for idx in np.nonzero(~in_vocab_mask)[0]:
+            entity = entities[idx]
+            self.vocab[entity] = Vocab(index=len(self.vocab), count=1)
+            self.index2entity.append(entity)
+
+        # add vectors for new entities
+        self.vectors = vstack((self.vectors, weights[~in_vocab_mask]))
+
+        # change vectors for in_vocab entities if `replace` flag is specified
+        if replace:
+            in_vocab_idxs = [self.vocab[entities[idx]].index for idx in np.nonzero(in_vocab_mask)[0]]
+            self.vectors[in_vocab_idxs] = weights[in_vocab_mask]
+
+    def __setitem__(self, entities, weights):
+        """Add entities and theirs vectors in a manual way.
+        If some entity is already in the vocabulary, old vector is replaced with the new one.
+        This method is alias for `add` with `replace=True`.
+
+        Parameters
+        ----------
+        entities : {str, list of str}
+            Entities specified by string tags.
+        weights: {list of numpy.ndarray, numpy.ndarray}
+            List of 1D np.array vectors or 2D np.array of vectors.
+
+        """
+        if not isinstance(entities, list):
+            entities = [entities]
+            weights = weights.reshape(1, -1)
+
+        self.add(entities, weights, replace=True)
+
     def __getitem__(self, entities):
         """
         Accept a single entity (string tag) or list of entities as input.
@@ -163,6 +222,7 @@ def __getitem__(self, entities):
 
         If a list, return designated tags' vector representations as a
         2D numpy array: #tags x #vector_size.
+
         """
         if isinstance(entities, string_types):
             # allow calls like trained_model['office'], as a shorthand for trained_model[['office']]


Test patch:
diff --git a/gensim/test/test_keyedvectors.py b/gensim/test/test_keyedvectors.py
--- a/gensim/test/test_keyedvectors.py
+++ b/gensim/test/test_keyedvectors.py
@@ -153,6 +153,78 @@ def test_wv_property(self):
         """Test that the deprecated `wv` property returns `self`. To be removed in v4.0.0."""
         self.assertTrue(self.vectors is self.vectors.wv)
 
+    def test_add_single(self):
+        """Test that adding entity in a manual way works correctly."""
+        entities = ['___some_entity{}_not_present_in_keyed_vectors___'.format(i) for i in range(5)]
+        vectors = [np.random.randn(self.vectors.vector_size) for _ in range(5)]
+
+        # Test `add` on already filled kv.
+        for ent, vector in zip(entities, vectors):
+            self.vectors.add(ent, vector)
+
+        for ent, vector in zip(entities, vectors):
+            self.assertTrue(np.allclose(self.vectors[ent], vector))
+
+        # Test `add` on empty kv.
+        kv = EuclideanKeyedVectors(self.vectors.vector_size)
+        for ent, vector in zip(entities, vectors):
+            kv.add(ent, vector)
+
+        for ent, vector in zip(entities, vectors):
+            self.assertTrue(np.allclose(kv[ent], vector))
+
+    def test_add_multiple(self):
+        """Test that adding a bulk of entities in a manual way works correctly."""
+        entities = ['___some_entity{}_not_present_in_keyed_vectors___'.format(i) for i in range(5)]
+        vectors = [np.random.randn(self.vectors.vector_size) for _ in range(5)]
+
+        # Test `add` on already filled kv.
+        vocab_size = len(self.vectors.vocab)
+        self.vectors.add(entities, vectors, replace=False)
+        self.assertEqual(vocab_size + len(entities), len(self.vectors.vocab))
+
+        for ent, vector in zip(entities, vectors):
+            self.assertTrue(np.allclose(self.vectors[ent], vector))
+
+        # Test `add` on empty kv.
+        kv = EuclideanKeyedVectors(self.vectors.vector_size)
+        kv[entities] = vectors
+        self.assertEqual(len(kv.vocab), len(entities))
+
+        for ent, vector in zip(entities, vectors):
+            self.assertTrue(np.allclose(kv[ent], vector))
+
+    def test_set_item(self):
+        """Test that __setitem__ works correctly."""
+        vocab_size = len(self.vectors.vocab)
+
+        # Add new entity.
+        entity = '___some_new_entity___'
+        vector = np.random.randn(self.vectors.vector_size)
+        self.vectors[entity] = vector
+
+        self.assertEqual(len(self.vectors.vocab), vocab_size + 1)
+        self.assertTrue(np.allclose(self.vectors[entity], vector))
+
+        # Replace vector for entity in vocab.
+        vocab_size = len(self.vectors.vocab)
+        vector = np.random.randn(self.vectors.vector_size)
+        self.vectors['war'] = vector
+
+        self.assertEqual(len(self.vectors.vocab), vocab_size)
+        self.assertTrue(np.allclose(self.vectors['war'], vector))
+
+        # __setitem__ on several entities.
+        vocab_size = len(self.vectors.vocab)
+        entities = ['war', '___some_new_entity1___', '___some_new_entity2___', 'terrorism', 'conflict']
+        vectors = [np.random.randn(self.vectors.vector_size) for _ in range(len(entities))]
+
+        self.vectors[entities] = vectors
+
+        self.assertEqual(len(self.vectors.vocab), vocab_size + 2)
+        for ent, vector in zip(entities, vectors):
+            self.assertTrue(np.allclose(self.vectors[ent], vector))
+
 
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)