Repository: piskvorky/gensim

Issue:
loading a vocab file doesn't seem to support unicode_errors
#### Problem description

Hi, thank you for providing gensim, which is an amzing lib, for us!

When I tried to read a vocab file generated by native word2vec command, I got an error.
```python
>>> model = gensim.models.KeyedVectors.load_word2vec_format("test_model",unicode_errors='replace',fvocab='test_vocab')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/models/keyedvectors.py", line 1498, in load_word2vec_format
    limit=limit, datatype=datatype)
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/models/utils_any2vec.py", line 338, in _load_word2vec_format
    word, count = utils.to_unicode(line).strip().split()
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/utils.py", line 359, in any2unicode
    return unicode(text, encoding, errors=errors)
UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 9-10: invalid continuation byte
```

The cause seems that there is no option to work around when some decoding errors happens in the vocab file loading process.

In Python3.x, unicode is replaced with str at the top of utils_any2vec.py.

The default option of str(), built-in function, is
```python
class str(object=b'', encoding='utf-8', errors='strict')
```

So, I propose to make a room to cover some decoding errors for loading a vocab file :)


#### Steps/code/corpus to reproduce

1. Install Anaconda on Ubuntu18.04.
```bash
curl -LO https://repo.anaconda.com/archive/Anaconda3-2019.07-Linux-x86_64.sh
bash Anaconda3-2019.07-Linux-x86_64.sh
```
2 Clone 'gensim'
```bash
git clone https://github.com/RaRe-Technologies/gensim.git
```
3. Install gensim
```bash
cd gensim/
python setup.py install
```
4. Run the following script with test files.
```python
import gensim
model = gensim.models.KeyedVectors.load_word2vec_format("test_model",unicode_errors='replace',fvocab='test_vocab')
```
5. You will get an error like this.
```python
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/models/keyedvectors.py", line 1498, in load_word2vec_format
    limit=limit, datatype=datatype)
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/models/utils_any2vec.py", line 338, in _load_word2vec_format
    word, count = utils.to_unicode(line).strip().split()
  File "/home/ubuntu/anaconda3/lib/python3.7/site-packages/gensim-3.8.0-py3.7-linux-x86_64.egg/gensim/utils.py", line 359, in any2unicode
    return unicode(text, encoding, errors=errors)
UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 9-10: invalid continuation byte
```

*Note: These are not generated by word2vec command. these files are for reproducing this error.*

#### Versions

Ubuntu 18.04


```python
>>> import platform; print(platform.platform())
Linux-4.15.0-1044-aws-x86_64-with-debian-buster-sid
>>> import sys; print("Python", sys.version)
Python 3.7.3 (default, Mar 27 2019, 22:11:17)
[GCC 7.3.0]
>>> import numpy; print("NumPy", numpy.__version__)
NumPy 1.17.0
>>> import scipy; print("SciPy", scipy.__version__)
SciPy 1.3.0
>>> import gensim; print("gensim", gensim.__version__)
gensim 3.8.0
>>> from gensim.models import word2vec;print("FAST_VERSION", word2vec.FAST_VERSION)
FAST_VERSION 1

```


[testdata.zip](https://github.com/RaRe-Technologies/gensim/files/3475521/testdata.zip)



Discussion:

Resolving patch:
diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py
--- a/gensim/models/utils_any2vec.py
+++ b/gensim/models/utils_any2vec.py
@@ -335,7 +335,7 @@ def _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8'
         counts = {}
         with utils.open(fvocab, 'rb') as fin:
             for line in fin:
-                word, count = utils.to_unicode(line).strip().split()
+                word, count = utils.to_unicode(line, errors=unicode_errors).strip().split()
                 counts[word] = int(count)
 
     logger.info("loading projection weights from %s", fname)


Test patch:
diff --git a/gensim/test/test_data/w2v_keyedvectors_load_test.modeldata b/gensim/test/test_data/w2v_keyedvectors_load_test.modeldata
new file mode 100644
--- /dev/null
+++ b/gensim/test/test_data/w2v_keyedvectors_load_test.modeldata
@@ -0,0 +1,3 @@
+2 3
+ありがとう� 0.6 0.6 0.6
+どういたしまして� 0.1 0.2 0.3
diff --git a/gensim/test/test_data/w2v_keyedvectors_load_test.vocab b/gensim/test/test_data/w2v_keyedvectors_load_test.vocab
new file mode 100644
--- /dev/null
+++ b/gensim/test/test_data/w2v_keyedvectors_load_test.vocab
@@ -0,0 +1,2 @@
+ありがとう� 123
+どういたしまして� 789
diff --git a/gensim/test/test_keyedvectors.py b/gensim/test/test_keyedvectors.py
--- a/gensim/test/test_keyedvectors.py
+++ b/gensim/test/test_keyedvectors.py
@@ -361,6 +361,45 @@ def test(self):
         self.assertTrue(vectors.word_vec('computer') is not None)
 
 
+class Word2VecKeyedVectorsTest(unittest.TestCase):
+    def setUp(self):
+        self.model_path = datapath("w2v_keyedvectors_load_test.modeldata")
+        self.vocab_path = datapath("w2v_keyedvectors_load_test.vocab")
+
+    def test_load_model_and_vocab_file_strict(self):
+        """Test loading model and voacab files which have decoding errors: strict mode"""
+        with self.assertRaises(UnicodeDecodeError):
+            gensim.models.KeyedVectors.load_word2vec_format(
+                self.model_path, fvocab=self.vocab_path, binary=False, unicode_errors="strict")
+
+    def test_load_model_and_vocab_file_replace(self):
+        """Test loading model and voacab files which have decoding errors: replace mode"""
+        model = gensim.models.KeyedVectors.load_word2vec_format(
+            self.model_path, fvocab=self.vocab_path, binary=False, unicode_errors="replace")
+        self.assertEqual(model.vocab[u'ありがとう�'].count, 123)
+        self.assertEqual(model.vocab[u'どういたしまして�'].count, 789)
+        self.assertEqual(model.vocab[u'ありがとう�'].index, 0)
+        self.assertEqual(model.vocab[u'どういたしまして�'].index, 1)
+        self.assertTrue(np.array_equal(
+            model.get_vector(u'ありがとう�'), np.array([.6, .6, .6], dtype=np.float32)))
+        self.assertTrue(np.array_equal(
+            model.get_vector(u'どういたしまして�'), np.array([.1, .2, .3], dtype=np.float32)))
+
+    def test_load_model_and_vocab_file_ignore(self):
+        """Test loading model and voacab files which have decoding errors: ignore mode"""
+        model = gensim.models.KeyedVectors.load_word2vec_format(
+            self.model_path, fvocab=self.vocab_path, binary=False, unicode_errors="ignore")
+        print(model.vocab.keys())
+        self.assertEqual(model.vocab[u'ありがとう'].count, 123)
+        self.assertEqual(model.vocab[u'どういたしまして'].count, 789)
+        self.assertEqual(model.vocab[u'ありがとう'].index, 0)
+        self.assertEqual(model.vocab[u'どういたしまして'].index, 1)
+        self.assertTrue(np.array_equal(
+            model.get_vector(u'ありがとう'), np.array([.6, .6, .6], dtype=np.float32)))
+        self.assertTrue(np.array_equal(
+            model.get_vector(u'どういたしまして'), np.array([.1, .2, .3], dtype=np.float32)))
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()