trec dataset unavailable

trec dataset unavailable
Hi
when I try to load the trec dataset I am getting these errors, thanks for your help

`datasets.load_dataset("trec",  split="train")
`
```
  File "<stdin>", line 1, in <module>
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File " /idiap/home/rkarimi/.cache/huggingface/modules/datasets_modules/datasets/trec/ca4248481ad244f235f4cf277186cad2ee8769f975119a2bbfc41b8932b88bd7/trec.py", line 140, in _split_generators
    dl_files = dl_manager.download_and_extract(_URLs)
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/idiap/user/rkarimi/libs/anaconda3/envs/internship/lib/python3.7/site-packages/datasets/utils/file_utils.py", line 477, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://cogcomp.org/Data/QA/QC/train_5500.label
```
This has been fixed in datasets 1.1.3


load_dataset('cnn_dalymail', '3.0.0') gives a 'Not a directory' error
```
from datasets import load_dataset
dataset = load_dataset('cnn_dailymail', '3.0.0')
```
Stack trace:
```
---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-6-2e06a8332652> in <module>()
      1 from datasets import load_dataset
----> 2 dataset = load_dataset('cnn_dailymail', '3.0.0')

5 frames

/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    608         download_config=download_config,
    609         download_mode=download_mode,
--> 610         ignore_verifications=ignore_verifications,
    611     )
    612

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    513                     if not downloaded_from_gcs:
    514                         self._download_and_prepare(
--> 515                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    516                         )
    517                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    568         split_dict = SplitDict(dataset_name=self.name)
    569         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 570         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    571
    572         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'
```
I have ran the code on Google Colab
I get the same error. It was fixed some days ago, but again it appears


load_dataset('cnn_dalymail', '3.0.0') gives a 'Not a directory' error
```
from datasets import load_dataset
dataset = load_dataset('cnn_dailymail', '3.0.0')
```
Stack trace:
```
---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-6-2e06a8332652> in <module>()
      1 from datasets import load_dataset
----> 2 dataset = load_dataset('cnn_dailymail', '3.0.0')

5 frames

/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    608         download_config=download_config,
    609         download_mode=download_mode,
--> 610         ignore_verifications=ignore_verifications,
    611     )
    612

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    513                     if not downloaded_from_gcs:
    514                         self._download_and_prepare(
--> 515                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    516                         )
    517                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    568         split_dict = SplitDict(dataset_name=self.name)
    569         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 570         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    571
    572         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'
```
I have ran the code on Google Colab
Hi @mrm8488 it's working again today without any fix so I am closing this issue.


load_dataset('cnn_dalymail', '3.0.0') gives a 'Not a directory' error
```
from datasets import load_dataset
dataset = load_dataset('cnn_dailymail', '3.0.0')
```
Stack trace:
```
---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-6-2e06a8332652> in <module>()
      1 from datasets import load_dataset
----> 2 dataset = load_dataset('cnn_dailymail', '3.0.0')

5 frames

/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    608         download_config=download_config,
    609         download_mode=download_mode,
--> 610         ignore_verifications=ignore_verifications,
    611     )
    612

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    513                     if not downloaded_from_gcs:
    514                         self._download_and_prepare(
--> 515                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    516                         )
    517                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    568         split_dict = SplitDict(dataset_name=self.name)
    569         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 570         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    571
    572         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'
```
I have ran the code on Google Colab
I see the issue happening again today -

[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
Downloading and preparing dataset cnn_dailymail/3.0.0 (download: 558.32 MiB, generated: 1.28 GiB, post-processed: Unknown size, total: 1.82 GiB) to /root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602...

---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-9-cd4bf8bea840> in <module>()
     22
     23
---> 24 train = load_dataset('cnn_dailymail', '3.0.0', split='train')
     25 validation = load_dataset('cnn_dailymail', '3.0.0', split='validation')
     26 test = load_dataset('cnn_dailymail', '3.0.0', split='test')

5 frames

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

Can someone please take a look ?


load_dataset('cnn_dalymail', '3.0.0') gives a 'Not a directory' error
```
from datasets import load_dataset
dataset = load_dataset('cnn_dailymail', '3.0.0')
```
Stack trace:
```
---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-6-2e06a8332652> in <module>()
      1 from datasets import load_dataset
----> 2 dataset = load_dataset('cnn_dailymail', '3.0.0')

5 frames

/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    608         download_config=download_config,
    609         download_mode=download_mode,
--> 610         ignore_verifications=ignore_verifications,
    611     )
    612

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    513                     if not downloaded_from_gcs:
    514                         self._download_and_prepare(
--> 515                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    516                         )
    517                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    568         split_dict = SplitDict(dataset_name=self.name)
    569         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 570         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    571
    572         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'
```
I have ran the code on Google Colab
Sometimes happens. Try in a while


load_dataset('cnn_dalymail', '3.0.0') gives a 'Not a directory' error
```
from datasets import load_dataset
dataset = load_dataset('cnn_dailymail', '3.0.0')
```
Stack trace:
```
---------------------------------------------------------------------------

NotADirectoryError                        Traceback (most recent call last)

<ipython-input-6-2e06a8332652> in <module>()
      1 from datasets import load_dataset
----> 2 dataset = load_dataset('cnn_dailymail', '3.0.0')

5 frames

/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    608         download_config=download_config,
    609         download_mode=download_mode,
--> 610         ignore_verifications=ignore_verifications,
    611     )
    612

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    513                     if not downloaded_from_gcs:
    514                         self._download_and_prepare(
--> 515                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    516                         )
    517                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    568         split_dict = SplitDict(dataset_name=self.name)
    569         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 570         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    571
    572         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'
```
I have ran the code on Google Colab
It is working now, thank you.


terminate called after throwing an instance of 'google::protobuf::FatalException'
Hi
I am using the dataset "iwslt2017-en-nl", and after downloading it I am getting this error when trying to evaluate it on T5-base with seq2seq_trainer.py in the huggingface repo could you assist me please? thanks

100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 63/63 [02:47<00:00,  2.18s/it][libprotobuf FATAL /sentencepiece/src/../third_party/protobuf-lite/google/protobuf/repeated_field.h:1505] CHECK failed: (index) >= (0):
terminate called after throwing an instance of 'google::protobuf::FatalException'
  what():  CHECK failed: (index) >= (0):
run_t5_base_eval.sh: line 19:  5795 Aborted
Loading the iwslt2017-en-nl config of iwslt2017 works fine on my side.
Maybe you can open an issue on transformers as well ? And also add more details about your environment (OS, python version, version of transformers and datasets etc.)


terminate called after throwing an instance of 'google::protobuf::FatalException'
Hi
I am using the dataset "iwslt2017-en-nl", and after downloading it I am getting this error when trying to evaluate it on T5-base with seq2seq_trainer.py in the huggingface repo could you assist me please? thanks

100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 63/63 [02:47<00:00,  2.18s/it][libprotobuf FATAL /sentencepiece/src/../third_party/protobuf-lite/google/protobuf/repeated_field.h:1505] CHECK failed: (index) >= (0):
terminate called after throwing an instance of 'google::protobuf::FatalException'
  what():  CHECK failed: (index) >= (0):
run_t5_base_eval.sh: line 19:  5795 Aborted
closing now, figured out this is because the max length of decoder was set smaller than the input_dimensions. thanks


[Feature Request] Add optional parameter in text loading script to preserve linebreaks
I'm working on a project about rhyming verse using phonetic poetry and song lyrics, and line breaks are a vital part of the data.

I recently switched over to use the datasets library when my various corpora grew larger than my computer's memory. And so far, it is SO great.

But the first time I processed all of my data into a dataset, I hadn't realized the text loader script was processing the source files line-by-line and stripping off the newlines.

Once I caught the issue, I made my own data loader by modifying one line in the default text loader (changing `batch = batch.splitlines()` to `batch = batch.splitlines(True)` inside `_generate_tables`). And so I'm all set as far as my project is concerned.

But if my use case is more general, it seems like it'd be pretty trivial to add a kwarg to the default text loader called keeplinebreaks or something, which would default to False and get passed to `splitlines()`.
Hi ! Thanks for your message.
Indeed it's a free feature we can add and that can be useful.
If you want to contribute, feel free to open a PR to add it to the text dataset script :)


OSCAR from Inria group

OSCAR from Inria group
## Adding a Dataset
- **Name:** *OSCAR* (Open Super-large Crawled ALMAnaCH coRpus), multilingual parsing of Common Crawl (separate crawls for many different languages), [here](https://oscar-corpus.com/).
- **Description:** *OSCAR or Open Super-large Crawled ALMAnaCH coRpus is a huge multilingual corpus obtained by language classification and filtering of the Common Crawl corpus using the goclassy architecture.*
- **Paper:** *[here](https://hal.inria.fr/hal-02148693)*
- **Data:** *[here](https://oscar-corpus.com/)*
- **Motivation:** *useful for unsupervised tasks in separate languages. In an ideal world, your team would be able to obtain the unshuffled version, that could be used to train GPT-2-like models (the shuffled version, I suppose, could be used for translation).*

I am aware that you do offer the "colossal" Common Crawl dataset already, but this has the advantage to be available in many subcorpora for different languages.

PR is already open here : #348
The only thing remaining is to compute the metadata of each subdataset (one per language + shuffled/unshuffled).
As soon as #863 is merged we can start computing them. This will take a bit of time though


OSCAR from Inria group

OSCAR from Inria group
## Adding a Dataset
- **Name:** *OSCAR* (Open Super-large Crawled ALMAnaCH coRpus), multilingual parsing of Common Crawl (separate crawls for many different languages), [here](https://oscar-corpus.com/).
- **Description:** *OSCAR or Open Super-large Crawled ALMAnaCH coRpus is a huge multilingual corpus obtained by language classification and filtering of the Common Crawl corpus using the goclassy architecture.*
- **Paper:** *[here](https://hal.inria.fr/hal-02148693)*
- **Data:** *[here](https://oscar-corpus.com/)*
- **Motivation:** *useful for unsupervised tasks in separate languages. In an ideal world, your team would be able to obtain the unshuffled version, that could be used to train GPT-2-like models (the shuffled version, I suppose, could be used for translation).*

I am aware that you do offer the "colossal" Common Crawl dataset already, but this has the advantage to be available in many subcorpora for different languages.

Grand, thanks for this!


Have Trouble importing `datasets`

Have Trouble importing `datasets`
I'm failing to import transformers (v4.0.0-dev), and tracing the cause seems to be failing to import datasets.

I cloned the newest version of datasets (master branch), and do `pip install -e .`.

Then, `import datasets` causes the error below.

```
~/workspace/Clone/datasets/src/datasets/utils/file_utils.py in <module>
    116 sys.path.append(str(HF_MODULES_CACHE))
    117
--> 118 os.makedirs(HF_MODULES_CACHE, exist_ok=True)
    119 if not os.path.exists(os.path.join(HF_MODULES_CACHE, "__init__.py")):
    120     with open(os.path.join(HF_MODULES_CACHE, "__init__.py"), "w"):

~/.pyenv/versions/anaconda3-2020.07/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    221             return
    222     try:
--> 223         mkdir(name, mode)
    224     except OSError:
    225         # Cannot rely on checking for EEXIST, since the operating system

FileNotFoundError: [Errno 2] No such file or directory: '<MY_HOME_DIRECTORY>/.cache/huggingface/modules'
```

The error occurs in `os.makedirs` in `file_utils.py`, even though `exist_ok = True` option is set.
(I use Python 3.8, so `exist_ok` is expected to work.)

I've checked some environment variables, and they are set as below.

```
*** NameError: name 'HF_MODULES_CACHE' is not defined
*** NameError: name 'hf_cache_home' is not defined
*** NameError: name 'XDG_CACHE_HOME' is not defined
```

Should I set some environment variables before using this library?
And, do you have any idea why "No such file or directory" occurs even though the `exist_ok = True` option is set?

Thank you in advance.
I'm sorry, this was a problem with my environment.
Now that I have identified the cause of environmental dependency, I would like to fix it and try it.
Excuse me for making a noise.


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
Same error here!


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
Same here! My kaggle notebook stopped working like yesterday. It's strange because I have fixed version of datasets==1.1.2


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
I'm looking at it right now


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
I couldn't reproduce unfortunately. I tried
```python
from datasets import load_dataset

load_dataset("cnn_dailymail", "3.0.0", download_mode="force_redownload")
```
and it worked fine on both my env (python 3.7.2) and colab (python 3.6.9)

Maybe there was an issue with the google drive download link of the dataset ?
Are you still having the issue ? If so could your give me more info about your python and requests version ?


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
No, It's working fine now. Very strange. Here are my python and request versions

requests         2.24.0
Python 3.8.2


Unable to download cnn_dailymail dataset
### Script to reproduce the error
```
from datasets import load_dataset

train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")
```

### Error
```
---------------------------------------------------------------------------
NotADirectoryError                        Traceback (most recent call last)
<ipython-input-8-47c39c228935> in <module>()
      1 from datasets import load_dataset
      2
----> 3 train_dataset = load_dataset("cnn_dailymail", "3.0.0", split= 'train[:10%')
      4 valid_dataset = load_dataset("cnn_dailymail","3.0.0", split="validation[:5%]")

5 frames
/usr/local/lib/python3.6/dist-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, script_version, **config_kwargs)
    609         download_config=download_config,
    610         download_mode=download_mode,
--> 611         ignore_verifications=ignore_verifications,
    612     )
    613

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    469                     if not downloaded_from_gcs:
    470                         self._download_and_prepare(
--> 471                             dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    472                         )
    473                     # Sync info

/usr/local/lib/python3.6/dist-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    524         split_dict = SplitDict(dataset_name=self.name)
    525         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 526         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    527
    528         # Checksums verification

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _split_generators(self, dl_manager)
    252     def _split_generators(self, dl_manager):
    253         dl_paths = dl_manager.download_and_extract(_DL_URLS)
--> 254         train_files = _subset_filenames(dl_paths, datasets.Split.TRAIN)
    255         # Generate shared vocabulary
    256

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _subset_filenames(dl_paths, split)
    153     else:
    154         logging.fatal("Unsupported split: %s", split)
--> 155     cnn = _find_files(dl_paths, "cnn", urls)
    156     dm = _find_files(dl_paths, "dm", urls)
    157     return cnn + dm

/root/.cache/huggingface/modules/datasets_modules/datasets/cnn_dailymail/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602/cnn_dailymail.py in _find_files(dl_paths, publisher, url_dict)
    132     else:
    133         logging.fatal("Unsupported publisher: %s", publisher)
--> 134     files = sorted(os.listdir(top_dir))
    135
    136     ret_files = []

NotADirectoryError: [Errno 20] Not a directory: '/root/.cache/huggingface/datasets/downloads/1bc05d24fa6dda2468e83a73cf6dc207226e01e3c48a507ea716dc0421da583b/cnn/stories'

```

Thanks for any suggestions.
It's working as expected.  Closing the issue

Thanks everybody.


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

The preprocessing tokenizes the input text. Tokenization outputs `input_ids`, `attention_mask`, `token_type_ids` and `special_tokens_mask`. All those are of length`max_seq_length` because of padding. Therefore for each sample it generate 4 *`max_seq_length` integers. Currently they're all saved as int64. This is why the tokenization takes so much space.

I'm sure we can optimize that though
What do you think @sgugger ?


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

First I think we should disable padding in the dataset processing and let the data collator do it.

Then I'm wondering if you need attention_mask and token_type_ids at this point ?

Finally we can also specify the output feature types at this line https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py#L280 to use more optimized integer precisions for the output. Maybe something like:
- input_ids: uint16 or uint32
- token_type_ids: uint8 or bool
- attention_mask: bool
- special_tokens_mask: bool

Also IMO these changes are all on the `transformers` side. Maybe we should discuss on the `transformers` repo


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

> First I think we should disable padding in the dataset processing and let the data collator do it.

No, you can't do that on TPUs as dynamic shapes will result in a very slow training. The script can however be tweaked to use the `PaddingDataCollator` with a fixed max length instead of dynamic batching.

For the other optimizations, they can be done by changing the script directly for each user's use case. Not sure we can find something that is general enough to be in transformers or the examples script.


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

Oh yes right..
Do you think that a lazy map feature on the `datasets` side could help to avoid storing padded tokenized texts then ?


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

I think I can do the tweak mentioned above with the data collator as short fix (but fully focused on v4 right now so that will be for later this week, beginning of next week :-) ).
If it doesn't hurt performance to tokenize on the fly, that would clearly be the long-term solution however!


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

> Hey guys,
>
> I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.
>
> I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?
>
> I've used the following CMD:
> `python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

It's actually because of the parameter 'preprocessing_num_worker' when using TPU.
I am also planning to have my model trained on the google TPU with a 11gb text corpus. With x8 cores enabled, each TPU core has its own dataset.  When not using distributed training, the preprocessed file is about 77gb. On the opposite, if enable xla, the file produced will easily consume all my free space(more than 220gb, I think it will be, in the end, around 600gb ).
So I think that's maybe where the problem came from.

Is there any possibility that all of the cores share the same preprocess dataset?

@sgugger @RammMaschine


Possible Bug: Small training/dataset file creates gigantic output
Hey guys,

I was trying to create a new bert model from scratch via _huggingface transformers + tokenizers + dataets_ (actually using this example script by your team: https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py). It was supposed to be a first test with a small 5 GB raw text file but I can't even end the preprocessing handled by datasets because this tiny 5 GB text file becomes more than 1 TB when processing. My system was running out of space and crashed prematurely.

I've done training from scratch via Google's bert repo in the past and I can remember that the resulting pretraining data can become quite big. But 5 GB becoming 1 TB was never the case. Is this considered normal or is it a bug?

I've used the following CMD:
`python xla_spawn.py --num_cores=8 run_mlm.py --model_type bert --config_name config.json --tokenizer_name tokenizer.json --train_file dataset_full.txt --do_train --output_dir out --max_steps 500000 --save_steps 2500 --save_total_limit 2 --prediction_loss_only --line_by_line --max_seq_length 128 --pad_to_max_length --preprocessing_num_workers 16 --per_device_train_batch_size 128 --overwrite_output_dir --debug`

Hi @NebelAI, we have optimized Datasets' disk usage in the latest release v1.5.

Feel free to update your Datasets version
```shell
pip install -U datasets
```
and see if it better suits your needs.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
Hi,I also posted it to the forum, but this is a bug, perhaps it needs to be reported here? thanks


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
It looks like the official OPUS server for WMT16 doesn't provide the data files anymore (503 error).
I searched a bit and couldn't find a mirror except maybe http://nlp.ffzg.hr/resources/corpora/setimes/ (the data are a cleaned version of the original ones though)
Should we consider replacing the old urls with these ones even though it's not the exact same data ?


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
The data storage is down at the moment. Sorry. Hopefully, it will come back soon. Apologies for the inconvenience ...


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
Dear great huggingface team, this is not working yet, I really appreciate some temporary fix on this, I need this for my project and this is time sensitive and I will be grateful for your help on this.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
We have reached out to the OPUS team which is currently working on making the data available again. Cc @jorgtied


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
thank you @thomwolf  and HuggingFace team for the help.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
OPUS is still down - hopefully back tomorrow.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
Hi, this is still down, I would be really grateful if you could ping them one more time. thank you so much.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
Hi
I am trying with multiple setting of wmt datasets and all failed so far, I need to have at least one dataset working for testing somecodes, and this is really time sensitive, I greatly appreciate letting me know of one translation datasets currently working. thanks


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
It is still down, unfortunately. I'm sorry for that. It should come up again later today or tomorrow at the latest if no additional complications will happen.


wmt16 does not download

wmt16 does not download
Hi, I appreciate your help with the following error, thanks

>>> from datasets import load_dataset
>>> dataset = load_dataset("wmt16", "ro-en", split="train")
Downloading and preparing dataset wmt16/ro-en (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/wmt16/ro-en/1.0.0/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308...
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/load.py", line 611, in load_dataset
    ignore_verifications=ignore_verifications,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 476, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/builder.py", line 531, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/root/.cache/huggingface/modules/datasets_modules/datasets/wmt16/7b2c4443a7d34c2e13df267eaa8cab4c62dd82f6b62b0d9ecc2e3a673ce17308/wmt_utils.py", line 755, in _split_generators
    downloaded_files = dl_manager.download_and_extract(urls_to_download)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 254, in download_and_extract
    return self.extract(self.download(url_or_urls))
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/download_manager.py", line 179, in download
    num_proc=download_config.num_proc,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in map_nested
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 225, in <listcomp>
    _single_map_nested((function, obj, types, None, True)) for obj in tqdm(iterable, disable=disable_tqdm)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in _single_map_nested
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 181, in <listcomp>
    mapped = [_single_map_nested((function, v, types, None, True)) for v in pbar]
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/py_utils.py", line 163, in _single_map_nested
    return function(data_struct)
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 308, in cached_path
    use_etag=download_config.use_etag,
  File "/root/anaconda3/envs/pytorch/lib/python3.6/site-packages/datasets/utils/file_utils.py", line 475, in get_from_cache
    raise ConnectionError("Couldn't reach {}".format(url))
ConnectionError: Couldn't reach http://opus.nlpl.eu/download.php?f=SETIMES/v2/tmx/en-ro.tmx.gz
Hi all,
I pulled a request that fix this issue by replacing urls.

https://github.com/huggingface/datasets/pull/1901

Thanks!