Remove checksum download in c4
State: closed | Author: lhoestq (MEMBER)

There was a line from the original tfds script that was still there and causing issues when loading the c4 script. This one should fix #233 and allow anyone to load the c4 script to generate the dataset

Comments: Commenting again in case [previous thread](https://github.com/huggingface/nlp/pull/233) was inactive.

@lhoestq I am facing `IsADirectoryError` while downloading with this command.
Can you pls look into it & help me.
I'm using version 0.4.0 of `nlp`.

```
dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')
```

Here's the complete stack trace.

```
Downloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, post-processed: Unknown sizetotal: Unknown size) to /home/devops/.cache/huggingface/datasets/c4/en/2.3.0/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7...

---------------------------------------------------------------------------
IsADirectoryError                         Traceback (most recent call last)
<ipython-input-11-f622e6705e03> in <module>
----> 1 dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)
    547     # Download and prepare data
    548     builder_instance.download_and_prepare(
--> 549         download_config=download_config, download_mode=download_mode, ignore_verifications=ignore_verifications,
    550     )
    551 

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    461                 if not downloaded_from_gcs:
    462                     self._download_and_prepare(
--> 463                         dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    464                     )
    465                 # Sync info

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos)
    964         pipeline = beam_utils.BeamPipeline(runner=beam_runner, options=beam_options,)
    965         super(BeamBasedBuilder, self)._download_and_prepare(
--> 966             dl_manager, verify_infos=False, pipeline=pipeline,
    967         )  # TODO handle verify_infos in beam datasets
    968         # Run pipeline

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    516         split_dict = SplitDict(dataset_name=self.name)
    517         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 518         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    519         # Checksums verification
    520         if verify_infos:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/datasets/c4/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7/c4.py in _split_generators(self, dl_manager, pipeline)
    187         if self.config.realnewslike:
    188             files_to_download["realnews_domains"] = _REALNEWS_DOMAINS_URL
--> 189         file_paths = dl_manager.download_and_extract(files_to_download)
    190 
    191         if self.config.webtextlike:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download_and_extract(self, url_or_urls)
    218             extracted_path(s): `str`, extracted paths of given URL(s).
    219         """
--> 220         return self.extract(self.download(url_or_urls))
    221 
    222     def get_recorded_sizes_checksums(self):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download(self, url_or_urls)
    156             lambda url: cached_path(url, download_config=self._download_config,), url_or_urls,
    157         )
--> 158         self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
    159         return downloaded_path_or_paths
    160 

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in _record_sizes_checksums(self, url_or_urls, downloaded_path_or_paths)
    106         flattened_downloaded_path_or_paths = flatten_nested(downloaded_path_or_paths)
    107         for url, path in zip(flattened_urls_or_urls, flattened_downloaded_path_or_paths):
--> 108             self._recorded_sizes_checksums[url] = get_size_checksum_dict(path)
    109 
    110     def download_custom(self, url_or_urls, custom_download):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/info_utils.py in get_size_checksum_dict(path)
     77     """Compute the file size and the sha256 checksum of a file"""
     78     m = sha256()
---> 79     with open(path, "rb") as f:
     80         for chunk in iter(lambda: f.read(1 << 20), b""):
     81             m.update(chunk)

IsADirectoryError: [Errno 21] Is a directory: '/'

```

Can anyone please try to see what I am doing wrong or is this a bug?