Unable to download the Chinese `wikipedia`, the dumpstatus.json not found!
Opened by beyondguo (closed)

### Describe the bug

I tried:
`data = load_dataset('wikipedia', '20220301.zh', beam_runner='DirectRunner')`
and
`data = load_dataset("wikipedia", language="zh", date="20220301", beam_runner='DirectRunner')`
but both got:
`FileNotFoundError: Couldn't find file at https://dumps.wikimedia.org/zhwiki/20220301/dumpstatus.json`

the full report is:
```
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-13-d07c5021090c> in <module>
      1 from datasets import load_dataset
      2 
----> 3 data = load_dataset("wikipedia", language="zh", date="20220301", beam_runner='DirectRunner')<?, ?it/s]

/opt/conda/lib/python3.8/site-packages/datasets/load.py in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, **config_kwargs)
   1740 
   1741     # Download and prepare data
-> 1742     builder_instance.download_and_prepare(
   1743         download_config=download_config,
   1744         download_mode=download_mode,

/opt/conda/lib/python3.8/site-packages/datasets/builder.py in download_and_prepare(self, output_dir, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, storage_options, **download_and_prepare_kwargs)
    812                             **download_and_prepare_kwargs,
    813                         }
--> 814                         self._download_and_prepare(
    815                             dl_manager=dl_manager,
    816                             verify_infos=verify_infos,

/opt/conda/lib/python3.8/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_splits_kwargs)
   1645             options=beam_options,
   1646         )
-> 1647         super()._download_and_prepare(
   1648             dl_manager, verify_infos=False, pipeline=pipeline, **prepare_splits_kwargs
   1649         )  # TODO handle verify_infos in beam datasets

/opt/conda/lib/python3.8/site-packages/datasets/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    881         split_dict = SplitDict(dataset_name=self.name)
    882         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 883         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    884 
    885         # Checksums verification

~/.cache/huggingface/modules/datasets_modules/datasets/wikipedia/aa542ed919df55cc5d3347f42dd4521d05ca68751f50dbc32bae2a7f1e167559/wikipedia.py in _split_generators(self, dl_manager, pipeline)
    943         info_url = _base_url(lang) + _INFO_FILE
    944         # Use dictionary since testing mock always returns the same result.
--> 945         downloaded_files = dl_manager.download_and_extract({"info": info_url})
    946 
    947         xml_urls = []

/opt/conda/lib/python3.8/site-packages/datasets/download/download_manager.py in download_and_extract(self, url_or_urls)
    431             extracted_path(s): `str`, extracted paths of given URL(s).
    432         """
--> 433         return self.extract(self.download(url_or_urls))
    434 
    435     def get_recorded_sizes_checksums(self):

/opt/conda/lib/python3.8/site-packages/datasets/download/download_manager.py in download(self, url_or_urls)
    308 
    309         start_time = datetime.now()
--> 310         downloaded_path_or_paths = map_nested(
    311             download_func,
    312             url_or_urls,

/opt/conda/lib/python3.8/site-packages/datasets/utils/py_utils.py in map_nested(function, data_struct, dict_only, map_list, map_tuple, map_numpy, num_proc, parallel_min_length, types, disable_tqdm, desc)
    427         num_proc = 1
    428     if num_proc <= 1 or len(iterable) < parallel_min_length:
--> 429         mapped = [
    430             _single_map_nested((function, obj, types, None, True, None))
    431             for obj in logging.tqdm(iterable, disable=disable_tqdm, desc=desc)

/opt/conda/lib/python3.8/site-packages/datasets/utils/py_utils.py in <listcomp>(.0)
    428     if num_proc <= 1 or len(iterable) < parallel_min_length:
    429         mapped = [
--> 430             _single_map_nested((function, obj, types, None, True, None))
    431             for obj in logging.tqdm(iterable, disable=disable_tqdm, desc=desc)
    432         ]

/opt/conda/lib/python3.8/site-packages/datasets/utils/py_utils.py in _single_map_nested(args)
    329     # Singleton first to spare some computation
    330     if not isinstance(data_struct, dict) and not isinstance(data_struct, types):
--> 331         return function(data_struct)
    332 
    333     # Reduce logging to keep things readable in multiprocessing with tqdm

/opt/conda/lib/python3.8/site-packages/datasets/download/download_manager.py in _download(self, url_or_filename, download_config)
    335             # append the relative path to the base_path
    336             url_or_filename = url_or_path_join(self._base_path, url_or_filename)
--> 337         return cached_path(url_or_filename, download_config=download_config)
    338 
    339     def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]):

/opt/conda/lib/python3.8/site-packages/datasets/utils/file_utils.py in cached_path(url_or_filename, download_config, **download_kwargs)
    186     if is_remote_url(url_or_filename):
    187         # URL, so get it from the cache (downloading if necessary)
--> 188         output_path = get_from_cache(
    189             url_or_filename,
    190             cache_dir=cache_dir,

/opt/conda/lib/python3.8/site-packages/datasets/utils/file_utils.py in get_from_cache(url, cache_dir, force_download, proxies, etag_timeout, resume_download, user_agent, local_files_only, use_etag, max_retries, use_auth_token, ignore_url_params, download_desc)
    533             )
    534         elif response is not None and response.status_code == 404:
--> 535             raise FileNotFoundError(f"Couldn't find file at {url}")
    536         _raise_if_offline_mode_is_enabled(f"Tried to reach {url}")
    537         if head_error is not None:

FileNotFoundError: Couldn't find file at https://dumps.wikimedia.org/zhwiki/20220301/dumpstatus.json
```



### Steps to reproduce the bug

`data = load_dataset('wikipedia', '20220301.zh', beam_runner='DirectRunner')`

### Expected behavior

download the data

### Environment info

python3.6
latest datasets/transformers version

Comments:
In the dumps page of the wiki (https://dumps.wikimedia.org/zhwiki/), I found the following dumps:
```
Index of /zhwiki/
[../](https://dumps.wikimedia.org/)
[20220701/](https://dumps.wikimedia.org/zhwiki/20220701/)                                          21-Aug-2022 01:48                   -
[20220720/](https://dumps.wikimedia.org/zhwiki/20220720/)                                          02-Sep-2022 01:48                   -
[20220801/](https://dumps.wikimedia.org/zhwiki/20220801/)                                          21-Sep-2022 01:44                   -
[20220820/](https://dumps.wikimedia.org/zhwiki/20220820/)                                          01-Oct-2022 09:39                   -
[20220901/](https://dumps.wikimedia.org/zhwiki/20220901/)                                          20-Oct-2022 09:44                   -
[20220920/](https://dumps.wikimedia.org/zhwiki/20220920/)                                          23-Sep-2022 12:06                   -
[20221001/](https://dumps.wikimedia.org/zhwiki/20221001/)                                          04-Oct-2022 15:10                   -
[20221020/](https://dumps.wikimedia.org/zhwiki/20221020/)                                          01-Nov-2022 03:15                   -
[latest/](https://dumps.wikimedia.org/zhwiki/latest/)                                            01-Nov-2022 03:15                   -
```

Maybe the older dumps are not available which caused the downloading failure? 

However, when I changed to the newer version:
```
data = load_dataset('wikipedia', '20220701.zh', beam_runner='DirectRunner')
```

it shows:
```
ValueError: BuilderConfig 20220701.zh not found. Available: ['20220301.aa', '20220301.ab', '20220301.ace', '20220301.ady', '20220301.af', '20220301.ak', '20220301.als', '20220301.am', '20220301.an', '20220301.ang', '20220301.ar', '20220301.arc', '20220301.arz', '20220301.as', '20220301.ast', '20220301.atj', '20220301.av', '20220301.ay', '20220301.az', '20220301.azb', '20220301.ba', '20220301.bar', '20220301.bat-smg', '20220301.bcl', '20220301.be', '20220301.be-x-old', '20220301.bg', '20220301.bh', '20220301.bi', '20220301.bjn', '20220301.bm', '20220301.bn', '20220301.bo', '20220301.bpy', '20220301.br', '20220301.bs', '20220301.bug', '20220301.bxr', '20220301.ca', '20220301.cbk-zam', '20220301.cdo', '20220301.ce', '20220301.ceb', '20220301.ch', '20220301.cho', '20220301.chr', '20220301.chy', '20220301.ckb', '20220301.co', '20220301.cr', '20220301.crh', '20220301.cs', '20220301.csb', '20220301.cu', '20220301.cv', '20220301.cy', '20220301.da', '20220301.de', '20220301.din', '20220301.diq', '20220301.dsb', '20220301.dty', '20220301.dv', '20220301.dz', '20220301.ee', '20220301.el', '20220301.eml', '20220301.en', '20220301.eo', '20220301.es', '20220301.et', '20220301.eu', '20220301.ext', '20220301.fa', '20220301.ff', '20220301.fi', '20220301.fiu-vro', '20220301.fj', '20220301.fo', '20220301.fr', '20220301.frp', '20220301.frr', '20220301.fur', '20220301.fy', '20220301.ga', '20220301.gag', '20220301.gan', '20220301.gd', '20220301.gl', '20220301.glk', '20220301.gn', '20220301.gom', '20220301.gor', '20220301.got', '20220301.gu', '20220301.gv', '20220301.ha', '20220301.hak', '20220301.haw', '20220301.he', '20220301.hi', '20220301.hif', '20220301.ho', '20220301.hr', '20220301.hsb', '20220301.ht', '20220301.hu', '20220301.hy', '20220301.ia', '20220301.id', '20220301.ie', '20220301.ig', '20220301.ii', '20220301.ik', '20220301.ilo', '20220301.inh', '20220301.io', '20220301.is', '20220301.it', '20220301.iu', '20220301.ja', '20220301.jam', '20220301.jbo', '20220301.jv', '20220301.ka', '20220301.kaa', '20220301.kab', '20220301.kbd', '20220301.kbp', '20220301.kg', '20220301.ki', '20220301.kj', '20220301.kk', '20220301.kl', '20220301.km', '20220301.kn', '20220301.ko', '20220301.koi', '20220301.krc', '20220301.ks', '20220301.ksh', '20220301.ku', '20220301.kv', '20220301.kw', '20220301.ky', '20220301.la', '20220301.lad', '20220301.lb', '20220301.lbe', '20220301.lez', '20220301.lfn', '20220301.lg', '20220301.li', '20220301.lij', '20220301.lmo', '20220301.ln', '20220301.lo', '20220301.lrc', '20220301.lt', '20220301.ltg', '20220301.lv', '20220301.mai', '20220301.map-bms', '20220301.mdf', '20220301.mg', '20220301.mh', '20220301.mhr', '20220301.mi', '20220301.min', '20220301.mk', '20220301.ml', '20220301.mn', '20220301.mr', '20220301.mrj', '20220301.ms', '20220301.mt', '20220301.mus', '20220301.mwl', '20220301.my', '20220301.myv', '20220301.mzn', '20220301.na', '20220301.nah', '20220301.nap', '20220301.nds', '20220301.nds-nl', '20220301.ne', '20220301.new', '20220301.ng', '20220301.nl', '20220301.nn', '20220301.no', '20220301.nov', '20220301.nrm', '20220301.nso', '20220301.nv', '20220301.ny', '20220301.oc', '20220301.olo', '20220301.om', '20220301.or', '20220301.os', '20220301.pa', '20220301.pag', '20220301.pam', '20220301.pap', '20220301.pcd', '20220301.pdc', '20220301.pfl', '20220301.pi', '20220301.pih', '20220301.pl', '20220301.pms', '20220301.pnb', '20220301.pnt', '20220301.ps', '20220301.pt', '20220301.qu', '20220301.rm', '20220301.rmy', '20220301.rn', '20220301.ro', '20220301.roa-rup', '20220301.roa-tara', '20220301.ru', '20220301.rue', '20220301.rw', '20220301.sa', '20220301.sah', '20220301.sat', '20220301.sc', '20220301.scn', '20220301.sco', '20220301.sd', '20220301.se', '20220301.sg', '20220301.sh', '20220301.si', '20220301.simple', '20220301.sk', '20220301.sl', '20220301.sm', '20220301.sn', '20220301.so', '20220301.sq', '20220301.sr', '20220301.srn', '20220301.ss', '20220301.st', '20220301.stq', '20220301.su', '20220301.sv', '20220301.sw', '20220301.szl', '20220301.ta', '20220301.tcy', '20220301.te', '20220301.tet', '20220301.tg', '20220301.th', '20220301.ti', '20220301.tk', '20220301.tl', '20220301.tn', '20220301.to', '20220301.tpi', '20220301.tr', '20220301.ts', '20220301.tt', '20220301.tum', '20220301.tw', '20220301.ty', '20220301.tyv', '20220301.udm', '20220301.ug', '20220301.uk', '20220301.ur', '20220301.uz', '20220301.ve', '20220301.vec', '20220301.vep', '20220301.vi', '20220301.vls', '20220301.vo', '20220301.wa', '20220301.war', '20220301.wo', '20220301.wuu', '20220301.xal', '20220301.xh', '20220301.xmf', '20220301.yi', '20220301.yo', '20220301.za', '20220301.zea', '20220301.zh', '20220301.zh-classical', '20220301.zh-min-nan', '20220301.zh-yue', '20220301.zu']
```

So I guess adding the latest dumps versions to the `BuilderConfig` may solve the problem? But how to add it?

Hi, @beyondguo, thanks for reporting.

You have all the information in the dataset card: https://huggingface.co/datasets/wikipedia

> Then, you can load any subset of Wikipedia per language and per date this way:
> ```python
> from datasets import load_dataset
> 
> load_dataset("wikipedia", language="sw", date="20220120", beam_runner=...) 
> ```
> where you can pass as beam_runner any Apache Beam supported runner for (distributed) data processing (see [here](https://beam.apache.org/documentation/runners/capability-matrix/)). Pass "DirectRunner" to run it on your machine.
> 
> You can find the full list of languages and dates [here](https://dumps.wikimedia.org/backup-index.html).

Note that you have to pass the language and date as keyword arguments, and the available dates depend on the language and can be found on Wikimedia website.

Also:
> Some subsets of Wikipedia have already been processed by HuggingFace, and you can load them just with:
> ```python
> load_dataset("wikipedia", "20220301.en")
> ```
> The list of pre-processed subsets is:
> - "20220301.de"
> - "20220301.en"
> - "20220301.fr"
> - "20220301.frr"
> - "20220301.it"
> - "20220301.simple"