Repository: piskvorky/gensim

Issue:
loading fastText model trained with pretrained_vectors still fails (see: #2350)
## Description
Loading pretrained `fastext_model.bin` with `gensim.models.fasttext.FastText.load_fasttext_format('wiki-news-300d-1M-subword.bin')` fails with `AssertionError: unexpected number of vectors` despite fix for #2350.

## Steps/Code/Corpus to Reproduce
first install `develop` branch with: `pip install --upgrade git+git://github.com/RaRe-Technologies/gensim@develop`, then:
```python
#dependencies 
import requests, zipfile, io
from gensim.models.fasttext import FastText

#download model
ft_url = 'https://s3-us-west-1.amazonaws.com/fasttext-vectors/wiki-news-300d-1M-subword.bin.zip'
r = requests.get(ft_url)
z = zipfile.ZipFile(io.BytesIO(r.content))
z.extractall()

#attempt load
mod = FastText.load_fasttext_format('wiki-news-300d-1M-subword.bin')
```

#### Expected Results
Loaded model.

#### Actual Results
```
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-29-a054256d6f88> in <module>
      1 #load model
      2 from gensim.models.fasttext import FastText
----> 3 mod = FastText.load_fasttext_format('wiki-news-300d-1M-subword.bin')
      4 # from gensim.models import KeyedVectors
      5 # wv = KeyedVectors.load_word2vec_format('wiki-news-300d-1M-subword.vec')

/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py in load_fasttext_format(cls, model_file, encoding, full_model)
   1012 
   1013         """
-> 1014         return _load_fasttext_format(model_file, encoding=encoding, full_model=full_model)
   1015 
   1016     def load_binary_data(self, encoding='utf8'):

/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py in _load_fasttext_format(model_file, encoding, full_model)
   1270     #
   1271     # We explicitly set min_count=1 regardless of the model's parameters to
-> 1272     # ignore the trim rule when building the vocabulary.  We do this in order
   1273     # to support loading native models that were trained with pretrained vectors.
   1274     # Such models will contain vectors for _all_ encountered words, not only

/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/keyedvectors.py in init_post_load(self, vectors, match_gensim)
   2205         """
   2206         vocab_words = len(self.vocab)
-> 2207         assert vectors.shape[0] == vocab_words + self.bucket, 'unexpected number of vectors'
   2208         assert vectors.shape[1] == self.vector_size, 'unexpected vector dimensionality'
   2209 

AssertionError: unexpected number of vectors
```

## Versions
Darwin-18.2.0-x86_64-i386-64bit
Python 3.7.2 (default, Dec 29 2018, 00:00:04) 
[Clang 4.0.1 (tags/RELEASE_401/final)]
NumPy 1.16.1
SciPy 1.2.0
gensim 3.7.1
FAST_VERSION 1

#### thanks for your work! 




Discussion:
Is there anything I can do to help this get looked at?
@cbjrobertson Thank you for reporting this.

> Is there anything I can do to help this get looked at?

One thing that would help is a reproducible example with a smaller model. The model you've linked to is several GB, if you could find a smaller model, it'd be easier (quicker) to reproduce the bug.

If not, then no big deal, and I'll try to have a look at it during the week.
> @cbjrobertson Thank you for reporting this.
> 
> > Is there anything I can do to help this get looked at?
> 
> One thing that would help is a reproducible example with a smaller model. The model you've linked to is several GB, if you could find a smaller model, it'd be easier (quicker) to reproduce the bug.
> 
> If not, then no big deal, and I'll try to have a look at it during the week.

Cool... so... here goes:

### Reproducible code:

```python
#dependencies 
import requests, zipfile, io, os
from gensim.models.fasttext import FastText


#download model
ft_url = 'https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M-subword.zip'
zpath = 'crawl-300d-2M-subword.zip'
mpath = 'crawl-300d-2M-subword.bin'

if os.path.isfile(mpath):
    #attempt load
    mod = FastText.load_fasttext_format(mpath,encoding='utf-8')
elif not os.path.isfile(zpath):
    r = requests.get(ft_url)
    z = zipfile.ZipFile(io.BytesIO(r.content))
    z.extractall()
else:
    z = zipfile.ZipFile(zpath)
    z.extractall()
    mod = FastText.load_fasttext_format(mpath,encoding='utf-8')
```
This is at least downloads from a faster source. However, it fails with: 

```
Traceback (most recent call last):

  File "<ipython-input-1-5ff598c5927b>", line 1, in <module>
    runfile('/Users/cole/Desktop/fast_text_test.py', wdir='/Users/cole/Desktop')

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/spyder_kernels/customize/spydercustomize.py", line 704, in runfile
    execfile(filename, namespace)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/spyder_kernels/customize/spydercustomize.py", line 108, in execfile
    exec(compile(f.read(), filename, 'exec'), namespace)

  File "/Users/cole/Desktop/fast_text_test.py", line 21, in <module>
    mod = FastText.load_fasttext_format(mpath,encoding='utf-8')

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py", line 1014, in load_fasttext_format
    return _load_fasttext_format(model_file, encoding=encoding, full_model=full_model)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py", line 1248, in _load_fasttext_format
    m = gensim.models._fasttext_bin.load(fin, encoding=encoding, full_model=full_model)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/_fasttext_bin.py", line 257, in load
    raw_vocab, vocab_size, nwords = _load_vocab(fin, new_format, encoding=encoding)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/_fasttext_bin.py", line 177, in _load_vocab
    word = word_bytes.decode(encoding)

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xce in position 57: unexpected end of data
```

I patched/hacked my way around that by adding the following two lines of code `gensim/models/_fasttext_bin.py`:

```
+ Line 35         from bs4 import UnicodeDammit
- Line 178        #word = word_bytes.decode(encoding)
+ Line 178        word = UnicodeDammit(word_bytes).unicode_markup
```

Which actually runs for a while, but then fails with: 
```
Traceback (most recent call last):

  File "<ipython-input-2-5ff598c5927b>", line 1, in <module>
    runfile('/Users/cole/Desktop/fast_text_test.py', wdir='/Users/cole/Desktop')

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/spyder_kernels/customize/spydercustomize.py", line 704, in runfile
    execfile(filename, namespace)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/spyder_kernels/customize/spydercustomize.py", line 108, in execfile
    exec(compile(f.read(), filename, 'exec'), namespace)

  File "/Users/cole/Desktop/fast_text_test.py", line 21, in <module>
    mod = FastText.load_fasttext_format(mpath,encoding='utf-8')

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py", line 1014, in load_fasttext_format
    return _load_fasttext_format(model_file, encoding=encoding, full_model=full_model)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/fasttext.py", line 1248, in _load_fasttext_format
    m = gensim.models._fasttext_bin.load(fin, encoding=encoding, full_model=full_model)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/_fasttext_bin.py", line 262, in load
    vectors_ngrams = _load_matrix(fin, new_format=new_format)

  File "/anaconda3/envs/tensor_env/lib/python3.7/site-packages/gensim/models/_fasttext_bin.py", line 225, in _load_matrix
    matrix = matrix.reshape((num_vectors, dim))

ValueError: cannot reshape array of size 179573500 into shape (4000000,300)
```

You may want to simply download the original link by hand. When I did it directly it was a lot faster... 





Resolving patch:
diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py
--- a/gensim/models/_fasttext_bin.py
+++ b/gensim/models/_fasttext_bin.py
@@ -30,11 +30,14 @@
 """
 
 import collections
+import io
 import logging
 import struct
 
 import numpy as np
 
+_END_OF_WORD_MARKER = b'\x00'
+
 logger = logging.getLogger(__name__)
 
 _FASTTEXT_FILEFORMAT_MAGIC = 793712314
@@ -168,13 +171,22 @@ def _load_vocab(fin, new_format, encoding='utf-8'):
 
     raw_vocab = collections.OrderedDict()
     for i in range(vocab_size):
-        word_bytes = b''
+        word_bytes = io.BytesIO()
         char_byte = fin.read(1)
-        # Read vocab word
-        while char_byte != b'\x00':
-            word_bytes += char_byte
+
+        while char_byte != _END_OF_WORD_MARKER:
+            word_bytes.write(char_byte)
             char_byte = fin.read(1)
-        word = word_bytes.decode(encoding)
+
+        word_bytes = word_bytes.getvalue()
+        try:
+            word = word_bytes.decode(encoding)
+        except UnicodeDecodeError:
+            word = word_bytes.decode(encoding, errors='ignore')
+            logger.error(
+                'failed to decode invalid unicode bytes %r; ignoring invalid characters, using %r',
+                word_bytes, word
+            )
         count, _ = _struct_unpack(fin, '@qb')
         raw_vocab[word] = count
 


Test patch:
diff --git a/gensim/test/test_fasttext.py b/gensim/test/test_fasttext.py
--- a/gensim/test/test_fasttext.py
+++ b/gensim/test/test_fasttext.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import division
 
+import io
 import logging
 import unittest
 import os
@@ -19,6 +20,8 @@
 from gensim.models.wrappers.fasttext import FastText as FT_wrapper
 from gensim.models.keyedvectors import Word2VecKeyedVectors
 from gensim.test.utils import datapath, get_tmpfile, temporary_file, common_texts as sentences
+import gensim.models._fasttext_bin
+
 
 try:
     from pyemd import emd  # noqa:F401
@@ -1209,6 +1212,69 @@ def test_out_of_vocab(self):
         self.assertRaises(KeyError, model.wv.word_vec, 'streamtrain')
 
 
+class UnicodeVocabTest(unittest.TestCase):
+    def test_ascii(self):
+        buf = io.BytesIO()
+        buf.name = 'dummy name to keep fasttext happy'
+        buf.write(struct.pack('@3i', 2, -1, -1))  # vocab_size, nwords, nlabels
+        buf.write(struct.pack('@1q', -1))
+        buf.write(b'hello')
+        buf.write(b'\x00')
+        buf.write(struct.pack('@qb', 1, -1))
+        buf.write(b'world')
+        buf.write(b'\x00')
+        buf.write(struct.pack('@qb', 2, -1))
+        buf.seek(0)
+
+        raw_vocab, vocab_size, nlabels = gensim.models._fasttext_bin._load_vocab(buf, False)
+        expected = {'hello': 1, 'world': 2}
+        self.assertEqual(expected, dict(raw_vocab))
+
+        self.assertEqual(vocab_size, 2)
+        self.assertEqual(nlabels, -1)
+
+    def test_bad_unicode(self):
+        buf = io.BytesIO()
+        buf.name = 'dummy name to keep fasttext happy'
+        buf.write(struct.pack('@3i', 2, -1, -1))  # vocab_size, nwords, nlabels
+        buf.write(struct.pack('@1q', -1))
+        #
+        # encountered in https://github.com/RaRe-Technologies/gensim/issues/2378
+        # The model from downloaded from
+        # https://s3-us-west-1.amazonaws.com/fasttext-vectors/wiki-news-300d-1M-subword.bin.zip
+        # suffers from bad characters in a few of the vocab terms.  The native
+        # fastText utility loads the model fine, but we trip up over the bad
+        # characters.
+        #
+        buf.write(
+            b'\xe8\x8b\xb1\xe8\xaa\x9e\xe7\x89\x88\xe3\x82\xa6\xe3\x82\xa3\xe3'
+            b'\x82\xad\xe3\x83\x9a\xe3\x83\x87\xe3\x82\xa3\xe3\x82\xa2\xe3\x81'
+            b'\xb8\xe3\x81\xae\xe6\x8a\x95\xe7\xa8\xbf\xe3\x81\xaf\xe3\x81\x84'
+            b'\xe3\x81\xa4\xe3\x81\xa7\xe3\x82\x82\xe6'
+        )
+        buf.write(b'\x00')
+        buf.write(struct.pack('@qb', 1, -1))
+        buf.write(
+            b'\xd0\xb0\xd0\xb4\xd0\xbc\xd0\xb8\xd0\xbd\xd0\xb8\xd1\x81\xd1\x82'
+            b'\xd1\x80\xd0\xb0\xd1\x82\xd0\xb8\xd0\xb2\xd0\xbd\xd0\xbe-\xd1\x82'
+            b'\xd0\xb5\xd1\x80\xd1\x80\xd0\xb8\xd1\x82\xd0\xbe\xd1\x80\xd0\xb8'
+            b'\xd0\xb0\xd0\xbb\xd1\x8c\xd0\xbd\xd1'
+        )
+        buf.write(b'\x00')
+        buf.write(struct.pack('@qb', 2, -1))
+        buf.seek(0)
+
+        raw_vocab, vocab_size, nlabels = gensim.models._fasttext_bin._load_vocab(buf, False)
+        expected = {
+            u'英語版ウィキペディアへの投稿はいつでも': 1,
+            u'административно-территориальн': 2,
+        }
+        self.assertEqual(expected, dict(raw_vocab))
+
+        self.assertEqual(vocab_size, 2)
+        self.assertEqual(nlabels, -1)
+
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()