Repository: piskvorky/gensim

Issue:
Unicode encode problem when saving doc2vec models in plain word2vec format.
#### Description
Exporting doc2vec embeddings in plain word2vec format fails when document title contains unicode chars that cannot be ASCII encoded. This only fails in Python 2.7 because Python 3 defaults to unicode strings. The problem is the call `str(..)` in [doc2vec.py](https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/doc2vec.py#L850)

#### Code to Reproduce
Example:
```
from gensim.models import doc2vec

# Load doc2vec model with a document title not ASCII encodable
m = Doc2Vec.load('model')
m.save_word2vec_format('model.out')
```

#### Expected Results
Export works

#### Actual Results
```
---------------------------------------------------------------------------
UnicodeEncodeError                        Traceback (most recent call last)
<ipython-input-3-d12938849014> in <module>()
----> 1 m.save_word2vec_format('model.out')

..../.virtualenvs/gensim27_2.3/local/lib/python2.7/site-packages/gensim-2.3.0-py2.7-linux-x86_64.egg/gensim/models/doc2vec.pyc in save_word2vec_format(self, fname, doctag_vec, word_vec, prefix, fvocab, binary)
    848                 # store as in input order
    849                 for i in range(len(self.docvecs)):
--> 850                     doctag = prefix + str(self.docvecs.index_to_doctag(i))
    851                     row = self.docvecs.doctag_syn0[i]
    852                     if binary:

UnicodeEncodeError: 'ascii' codec can't encode character u'\xf3' in position 8: ordinal not in range(128)
```

#### Versions
Linux-4.4.0-59-generic-x86_64-with-Ubuntu-16.04-xenial
('Python', '2.7.12 (default, Nov 19 2016, 06:48:10) \n[GCC 5.4.0 20160609]')
('NumPy', '1.13.1')
('SciPy', '0.19.1')
('gensim', '2.3.0')
('FAST_VERSION', 1)


Discussion:

Resolving patch:
diff --git a/gensim/models/doc2vec.py b/gensim/models/doc2vec.py
--- a/gensim/models/doc2vec.py
+++ b/gensim/models/doc2vec.py
@@ -847,7 +847,7 @@ def save_word2vec_format(self, fname, doctag_vec=False, word_vec=True, prefix='*
                     fout.write(utils.to_utf8("%s %s\n" % (total_vec, self.vector_size)))
                 # store as in input order
                 for i in range(len(self.docvecs)):
-                    doctag = prefix + str(self.docvecs.index_to_doctag(i))
+                    doctag = u"%s%s" % (prefix, self.docvecs.index_to_doctag(i))
                     row = self.docvecs.doctag_syn0[i]
                     if binary:
                         fout.write(utils.to_utf8(doctag) + b" " + row.tostring())


Test patch:
diff --git a/gensim/test/test_doc2vec.py b/gensim/test/test_doc2vec.py
--- a/gensim/test/test_doc2vec.py
+++ b/gensim/test/test_doc2vec.py
@@ -30,11 +30,16 @@
 
 
 class DocsLeeCorpus(object):
-    def __init__(self, string_tags=False):
+    def __init__(self, string_tags=False, unicode_tags=False):
         self.string_tags = string_tags
+        self.unicode_tags = unicode_tags
 
     def _tag(self, i):
-        return i if not self.string_tags else '_*%d' % i
+        if self.unicode_tags:
+            return u'_\xa1_%d' % i
+        elif self.string_tags:
+            return '_*%d' % i
+        return i
 
     def __iter__(self):
         with open(datapath('lee_background.cor')) as f:
@@ -95,6 +100,14 @@ def testPersistenceWord2VecFormat(self):
         binary_model_dv = keyedvectors.KeyedVectors.load_word2vec_format(test_word, binary=True)
         self.assertEqual(len(model.wv.vocab), len(binary_model_dv.vocab))
 
+    def test_unicode_in_doctag(self):
+        """Test storing document vectors of a model with unicode titles."""
+        model = doc2vec.Doc2Vec(DocsLeeCorpus(unicode_tags=True), min_count=1)
+        try:
+            model.save_word2vec_format(testfile(), doctag_vec=True, word_vec=True, binary=True)
+        except UnicodeEncodeError:
+            self.fail('Failed storing unicode title.')
+
     def test_load_mmap(self):
         """Test storing/loading the entire model."""
         model = doc2vec.Doc2Vec(sentences, min_count=1)