Repository: pytorch/text

Issue:
Loading pre-trained word vectors is broken
The logic to load the pre-trained word vectors seems to be broken.  I looked in `test/vocab.py` but didn't see any tests that covered correctness of loaded vectors.

Here's a snippet to reproduce the issue:

```python
from torchtext.vocab import Vocab 
from collections import Counter

the_vocab = Vocab(Counter(['the']), vectors='glove.6B.50d')

the_index = the_vocab.stoi['the']
print(the_index)
print(the_vocab.vectors.numpy()[the_index, :])
```

With the master branch this displays:

```
2
[  3.02532905e-17   1.40129846e-45   2.10194770e-44   0.00000000e+00
   1.00185045e-16   1.40129846e-45   1.40129846e-44   0.00000000e+00
   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
   4.84570079e-17   1.40129846e-45   1.54142831e-44   0.00000000e+00
   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00
   0.00000000e+00  -2.00000000e+00   8.72027897e-07  -4.65774808e-10
   4.82658813e-26   1.40129846e-45   0.00000000e+00   0.00000000e+00
   0.00000000e+00   0.00000000e+00  -8.29709032e-27   3.23318939e+02
   9.88949980e-32   1.40129846e-45   1.58111748e-32   1.40129846e-45
  -8.15662628e+28  -9.19432299e+27   9.88943397e-32   1.40129846e-45
   7.55058708e-26   1.40129846e-45]
```

However, if I manually download and unpack the GloVe 50d file, this is the line with `the` token:

```
the 0.418 0.24968 -0.41242 0.1217 0.34527 -0.044457 -0.49688 -0.17862 -0.00066023 -0.6566 0.27843 -0.14767 -0.55677 0.14658 -0.0095095 0.011658 0.10204 -0.12792 -0.8443 -0.12181 -0.016801 -0.33279 -0.1552 -0.23131 -0.19181 -1.8823 -0.76746 0.099051 -0.42125 -0.19526 4.0071 -0.18594 -0.52287 -0.31681 0.00059213 0.0074449 0.17778 -0.15897 0.012041 -0.054223 -0.29871 -0.15749 -0.34758 -0.045637 -0.44251 0.18785 0.0027849 -0.18411 -0.11514 -0.78581
```




Discussion:

Resolving patch:
diff --git a/torchtext/datasets/imdb.py b/torchtext/datasets/imdb.py
--- a/torchtext/datasets/imdb.py
+++ b/torchtext/datasets/imdb.py
@@ -43,7 +43,8 @@ def download(cls, root):
         if not os.path.isdir(path):
             fpath = os.path.join(path, cls.filename)
             if not os.path.isfile(fpath):
-                os.makedirs(os.path.dirname(fpath), exist_ok=True)
+                if not os.path.exists(os.path.dirname(fpath)):
+                    os.makedirs(os.path.dirname(fpath))
                 print('downloading {}'.format(cls.filename))
                 urllib.request.urlretrieve(os.path.join(cls.url, cls.filename), fpath)
             with tarfile.open(fpath, 'r:gz') as tar:
diff --git a/torchtext/datasets/trec.py b/torchtext/datasets/trec.py
--- a/torchtext/datasets/trec.py
+++ b/torchtext/datasets/trec.py
@@ -49,7 +49,8 @@ def download(cls, root):
             for fn in [cls.train_filename, cls.test_filename]:
                 fpath = os.path.join(path, fn)
                 if not os.path.isfile(fpath):
-                    os.makedirs(os.path.dirname(fpath), exist_ok=True)
+                    if not os.path.exists(os.path.dirname(fpath)):
+                        os.makedirs(os.path.dirname(fpath))
                     print('downloading {}'.format(fn))
                     urllib.request.urlretrieve(os.path.join(cls.url_base, fn), fpath)
         return os.path.join(path, '')
diff --git a/torchtext/vocab.py b/torchtext/vocab.py
--- a/torchtext/vocab.py
+++ b/torchtext/vocab.py
@@ -116,9 +116,9 @@ def load_vectors(self, vectors, unk_init=torch.Tensor.zero_, expand_vocab=False)
         self.vectors = torch.Tensor(len(self), tot_dim)
         for i, token in enumerate(self.itos):
             start_dim = 0
-            for i, v in enumerate(vectors):
-                end_dim = start_dim + vecs[i].dim
-                self.vectors[i][start_dim:end_dim] = vecs[i][token]
+            for j, v in enumerate(vectors):
+                end_dim = start_dim + vecs[j].dim
+                self.vectors[i][start_dim:end_dim] = vecs[j][token]
                 start_dim = end_dim
             assert(start_dim == tot_dim)
 
@@ -149,12 +149,13 @@ def vector_cache(self, url, root, fname):
         fname_pt = fname + '.pt'
         fname_txt = fname + '.txt'
         desc = os.path.basename(fname)
-        dest = os.path.join(root, os.path.basename(url))
 
         if not os.path.isfile(fname_pt):
+            dest = os.path.join(root, os.path.basename(url))
             if not os.path.isfile(fname_txt):
                 print('downloading vectors from {}'.format(url))
-                os.makedirs(root, exist_ok=True)
+                if not os.path.exists(root):
+                    os.makedirs(root)
                 with tqdm(unit='B', unit_scale=True, miniters=1, desc=desc) as t:
                     urlretrieve(url, dest, reporthook=reporthook(t))
                 print('extracting vectors into {}'.format(root))
@@ -204,7 +205,8 @@ class GloVe(Vectors):
         'glove.42B': 'http://nlp.stanford.edu/data/glove.42B.300d.zip',
         'glove.840B': 'http://nlp.stanford.edu/data/glove.840B.300d.zip',
         'glove.twitter.27B': 'http://nlp.stanford.edu/data/glove.twitter.27B.zip',
-        'glove.6B': 'http://nlp.stanford.edu/data/glove.6B.zip'
+        'glove.6B': 'http://nlp.stanford.edu/data/glove.6B.zip',
+        'glove.test_twitter.27B': None,
     }
 
     def __init__(self, root='.vector_cache', name='840B', dim=300, **kwargs):


Test patch:
diff --git a/.vector_cache/glove.test_twitter.27B.200d.pt b/.vector_cache/glove.test_twitter.27B.200d.pt
new file mode 100644
Binary files /dev/null and b/.vector_cache/glove.test_twitter.27B.200d.pt differ
diff --git a/.vector_cache/make_test_glove_vectors.py b/.vector_cache/make_test_glove_vectors.py
new file mode 100644
--- /dev/null
+++ b/.vector_cache/make_test_glove_vectors.py
@@ -0,0 +1,19 @@
+
+import torch
+import numpy as np
+
+if __name__ == '__main__':
+    vocab = ['The', 'the', 'hello', 'world', 'Hello', ',', '.']
+    dim = 200
+
+    stoi = {word: i for i, word in enumerate(vocab)}
+
+    vectors = torch.Tensor(len(vocab), dim)
+    with open('glove.twitter.27B.200d.txt', 'r') as fin:
+        for line in fin:
+            columns = line.strip().split()
+            if columns[0] in stoi:
+                vectors[stoi[columns[0]], :] = torch.from_numpy(
+                    np.array([float(ele) for ele in columns[1:]])
+                )
+    torch.save((stoi, vectors, dim), 'glove.test_twitter.27B.200d.pt')
diff --git a/test/test_vocab.py b/test/test_vocab.py
new file mode 100644
--- /dev/null
+++ b/test/test_vocab.py
@@ -0,0 +1,33 @@
+import numpy as np
+import unittest
+
+from torchtext import vocab
+from collections import Counter
+
+
+class TestVocab(unittest.TestCase):
+    def test_vocab(self):
+        c = Counter(['hello', 'world'])
+        v = vocab.Vocab(c, vectors='glove.test_twitter.27B.200d')
+
+        self.assertEqual(v.itos, ['<unk>', '<pad>', 'hello', 'world'])
+        vectors = v.vectors.numpy()
+
+        # The first 5 entries in each vector.
+        expected_glove_twitter = {
+            'hello': [0.34683, -0.19612, -0.34923, -0.28158, -0.75627],
+            'world': [0.035771, 0.62946, 0.27443, -0.36455, 0.39189],
+        }
+
+        for word in ['hello', 'world']:
+            self.assertTrue(
+                np.allclose(
+                    vectors[v.stoi[word], :5], expected_glove_twitter[word]
+                )
+            )
+
+        self.assertTrue(np.allclose(vectors[v.stoi['<unk>'], :], np.zeros(200)))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/vocab.py b/test/vocab.py
deleted file mode 100644
--- a/test/vocab.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from torchtext import vocab
-
-from collections import Counter
-c = Counter(['hello', 'world'])
-v = vocab.Vocab(c, vectors=['glove.twitter.27B.200d', 'charngram.100d'])
-print(v.itos)
-print(v.vectors)