OSError: Cannot find data file when not using the dummy dataset in RAG 
 ## Environment info

    transformers version: 3.3.1
    Platform: Linux-4.19
    Python version: 3.7.7
    PyTorch version (GPU?): 1.6.0
    Tensorflow version (GPU?): No
    Using GPU in script?: Yes
    Using distributed or parallel set-up in script?: No

## To reproduce

Steps to reproduce the behaviour:
```
import os
os.environ['HF_DATASETS_CACHE'] = '/workspace/notebooks/POCs/cache'

from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration

tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq")
retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=False) 
```

Plese note that I'm using the whole dataset: **use_dummy_dataset=False**
After around 4 hours (downloading and some other things) this is returned:

```
Downloading and preparing dataset wiki_dpr/psgs_w100.nq.exact (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /workspace/notebooks/POCs/cache/wiki_dpr/psgs_w100.nq.exact/0.0.0/14b973bf2a456087ff69c0fd34526684eed22e48e0dfce4338f9a22b965ce7c2...

---------------------------------------------------------------------------
UnpicklingError                           Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
    459             try:
--> 460                 return pickle.load(fid, **pickle_kwargs)
    461             except Exception:

UnpicklingError: pickle data was truncated

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    552                 # Prepare split will record examples associated to the split
--> 553                 self._prepare_split(split_generator, **prepare_split_kwargs)
    554             except OSError:

/opt/conda/lib/python3.7/site-packages/datasets/builder.py in _prepare_split(self, split_generator)
    840             for key, record in utils.tqdm(
--> 841                 generator, unit=" examples", total=split_info.num_examples, leave=False, disable=not_verbose
    842             ):

/opt/conda/lib/python3.7/site-packages/tqdm/notebook.py in __iter__(self, *args, **kwargs)
    217         try:
--> 218             for obj in super(tqdm_notebook, self).__iter__(*args, **kwargs):
    219                 # return super(tqdm...) will not catch exception

/opt/conda/lib/python3.7/site-packages/tqdm/std.py in __iter__(self)
   1128         try:
-> 1129             for obj in iterable:
   1130                 yield obj

~/.cache/huggingface/modules/datasets_modules/datasets/wiki_dpr/14b973bf2a456087ff69c0fd34526684eed22e48e0dfce4338f9a22b965ce7c2/wiki_dpr.py in _generate_examples(self, data_file, vectors_files)
    131                         break
--> 132                     vecs = np.load(open(vectors_files.pop(0), "rb"), allow_pickle=True)
    133                     vec_idx = 0

/opt/conda/lib/python3.7/site-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
    462                 raise IOError(
--> 463                     "Failed to interpret file %s as a pickle" % repr(file))
    464     finally:

OSError: Failed to interpret file <_io.BufferedReader name='/workspace/notebooks/POCs/cache/downloads/f34d5f091294259b4ca90e813631e69a6ded660d71b6cbedf89ddba50df94448'> as a pickle

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
<ipython-input-10-f28df370ac47> in <module>
      1 # ln -s /workspace/notebooks/POCs/cache /root/.cache/huggingface/datasets
----> 2 retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=False)

/opt/conda/lib/python3.7/site-packages/transformers/retrieval_rag.py in from_pretrained(cls, retriever_name_or_path, **kwargs)
    307         generator_tokenizer = rag_tokenizer.generator
    308         return cls(
--> 309             config, question_encoder_tokenizer=question_encoder_tokenizer, generator_tokenizer=generator_tokenizer
    310         )
    311 

/opt/conda/lib/python3.7/site-packages/transformers/retrieval_rag.py in __init__(self, config, question_encoder_tokenizer, generator_tokenizer)
    298         self.config = config
    299         if self._init_retrieval:
--> 300             self.init_retrieval()
    301 
    302     @classmethod

/opt/conda/lib/python3.7/site-packages/transformers/retrieval_rag.py in init_retrieval(self)
    324 
    325         logger.info("initializing retrieval")
--> 326         self.index.init_index()
    327 
    328     def postprocess_docs(self, docs, input_strings, prefix, n_docs, return_tensors=None):

/opt/conda/lib/python3.7/site-packages/transformers/retrieval_rag.py in init_index(self)
    238                 split=self.dataset_split,
    239                 index_name=self.index_name,
--> 240                 dummy=self.use_dummy_dataset,
    241             )
    242             self.dataset.set_format("numpy", columns=["embeddings"], output_all_columns=True)

/opt/conda/lib/python3.7/site-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613 

/opt/conda/lib/python3.7/site-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    474                     if not downloaded_from_gcs:
    475                         self._download_and_prepare(
--> 476                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    477                         )
    478                     # Sync info

/opt/conda/lib/python3.7/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    553                 self._prepare_split(split_generator, **prepare_split_kwargs)
    554             except OSError:
--> 555                 raise OSError("Cannot find data file. " + (self.manual_download_instructions or ""))
    556 
    557         if verify_infos:

OSError: Cannot find data file. 

```

Thanks 
 
 Same issue here. I will be digging further, but it looks like the [script](https://github.com/huggingface/datasets/blob/master/datasets/wiki_dpr/wiki_dpr.py#L132) is attempting to open a file that is not downloaded yet. 

```
99dcbca09109e58502e6b9271d4d3f3791b43f61f3161a76b25d2775ab1a4498.lock
```

```
---------------------------------------------------------------------------
UnpicklingError                           Traceback (most recent call last)
~/anaconda3/envs/eqa/lib/python3.7/site-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
    446             try:
--> 447                 return pickle.load(fid, **pickle_kwargs)
    448             except Exception:

UnpicklingError: pickle data was truncated

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
~/src/datasets/src/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    559 
--> 560         if verify_infos:
    561             verify_splits(self.info.splits, split_dict)

~/src/datasets/src/datasets/builder.py in _prepare_split(self, split_generator)
    847                 writer.write(example)
--> 848         finally:
    849             num_examples, num_bytes = writer.finalize()

~/anaconda3/envs/eqa/lib/python3.7/site-packages/tqdm/notebook.py in __iter__(self, *args, **kwargs)
    227         try:
--> 228             for obj in super(tqdm_notebook, self).__iter__(*args, **kwargs):
    229                 # return super(tqdm...) will not catch exception

~/anaconda3/envs/eqa/lib/python3.7/site-packages/tqdm/std.py in __iter__(self)
   1132         try:
-> 1133             for obj in iterable:
   1134                 yield obj

/hdd/rag/cache/huggingface/modules/datasets_modules/datasets/wiki_dpr/14b973bf2a456087ff69c0fd34526684eed22e48e0dfce4338f9a22b965ce7c2/wiki_dpr.py in _generate_examples(self, data_file, vectors_files)
    131                         break
--> 132                     vecs = np.load(open(vectors_files.pop(0), "rb"), allow_pickle=True)
    133                     vec_idx = 0

~/anaconda3/envs/eqa/lib/python3.7/site-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
    449                 raise IOError(
--> 450                     "Failed to interpret file %s as a pickle" % repr(file))
    451 

OSError: Failed to interpret file <_io.BufferedReader name='/hdd/rag/downloads/99dcbca09109e58502e6b9271d4d3f3791b43f61f3161a76b25d2775ab1a4498'> as a pickle

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
<ipython-input-8-24351ff8ce44> in <module>
      4 retriever = RagRetriever.from_pretrained("facebook/rag-sequence-nq", 
      5                                          index_name="exact",
----> 6                                          use_dummy_dataset=False)

~/src/transformers/src/transformers/retrieval_rag.py in from_pretrained(cls, retriever_name_or_path, **kwargs)
    321         generator_tokenizer = rag_tokenizer.generator
    322         return cls(
--> 323             config, question_encoder_tokenizer=question_encoder_tokenizer, generator_tokenizer=generator_tokenizer
    324         )
    325 

~/src/transformers/src/transformers/retrieval_rag.py in __init__(self, config, question_encoder_tokenizer, generator_tokenizer)
    310         self.config = config
    311         if self._init_retrieval:
--> 312             self.init_retrieval()
    313 
    314     @classmethod

~/src/transformers/src/transformers/retrieval_rag.py in init_retrieval(self)
    338 
    339         logger.info("initializing retrieval")
--> 340         self.index.init_index()
    341 
    342     def postprocess_docs(self, docs, input_strings, prefix, n_docs, return_tensors=None):

~/src/transformers/src/transformers/retrieval_rag.py in init_index(self)
    248                 split=self.dataset_split,
    249                 index_name=self.index_name,
--> 250                 dummy=self.use_dummy_dataset,
    251             )
    252             self.dataset.set_format("numpy", columns=["embeddings"], output_all_columns=True)

~/src/datasets/src/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    615     builder_instance.download_and_prepare(
    616         download_config=download_config,
--> 617         download_mode=download_mode,
    618         ignore_verifications=ignore_verifications,
    619     )

~/src/datasets/src/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    481                     # Sync info
    482                     self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
--> 483                     self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
    484                     self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
    485                     # Save info

~/src/datasets/src/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    560         if verify_infos:
    561             verify_splits(self.info.splits, split_dict)
--> 562 
    563         # Update the info object with the splits.
    564         self.info.splits = split_dict

OSError: Cannot find data file.
```

Thank you.