add script for downloading delta release (currently for cv12)

download_delta.py

@@ -0,0 +1,101 @@
+import urllib
+import sys
+import requests
+import os
+import logging
+import shutil
+import json
+from tqdm import tqdm
+import time
+from pathlib import Path
+from datasets.download import DownloadConfig, DownloadManager
+
+
+logging.basicConfig(
+    format='%(asctime)s %(levelname)s: %(message)s',
+    level=logging.INFO,
+    handlers=[
+        logging.FileHandler("cv12_download.log"),
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+
+_BUNDLE_URL_TEMPLATE_DELTA = 'cv-corpus-12.0-delta-2022-12-07/cv-corpus-12.0-delta-2022-12-07-{locale}.tar.gz'
+_BUNDLE_VERSION = _BUNDLE_URL_TEMPLATE_DELTA.split("/")[0]
+_API_URL = "https://commonvoice.mozilla.org/api/v1"
+
+
+def _get_bundle_url(locale, url_template):
+    path = url_template.replace("{locale}", locale)
+    path = urllib.parse.quote(path.encode("utf-8"), safe="~()*!.'")
+    response = requests.get(f"{_API_URL}/bucket/dataset/{path}", timeout=10.0).json()
+    return response["url"]
+
+
+def _log_download(locale, bundle_version):
+    email = "polina@huggingface.co"
+    payload = {"email": email, "locale": locale, "dataset": bundle_version}
+    requests.post(f"{_API_URL}/{locale}/downloaders", json=payload).json()
+
+
+def download_language(dl_manager, lang, root_dir):
+    _log_download(lang, _BUNDLE_VERSION)
+    url = _get_bundle_url(lang, _BUNDLE_URL_TEMPLATE_DELTA)
+    i = 1
+    while url == "https://s3.dualstack.us-west-2.amazonaws.com/":
+        if i == 6:
+            raise ConnectionError(f"Cannot download '{lang.upper()}' data, fetched url: {url}. ")
+        i += 1
+        logging.warning(f"Unsuccessful attempt to fetch data url. Trying {i} time. ")
+        time.sleep(15)
+        _log_download(lang, _BUNDLE_VERSION)
+        url = _get_bundle_url(lang, _BUNDLE_URL_TEMPLATE_DELTA)
+
+    logging.info(f"Trying to download data for '{lang.upper()}'... ")
+    path = dl_manager.download_and_extract(url)
+    if os.path.isdir(path):
+        logging.info(f"'{lang.upper()}' data downloaded to {path}. ")
+        shutil.move(path, root_dir / f"data/{lang}")
+    else:  # if it's not a dir, there was no data update in the release
+        logging.info(f"No data for '{lang.upper()}' found. ")
+
+
+def main():
+    root_dir = Path("")
+    with open("langs.json", "r") as f:
+        languages = json.load(f).keys()
+
+    if (root_dir / "langs_ok.txt").exists():
+        with open(root_dir / "langs_ok.txt") as f:
+            langs_to_skip = set([line.strip().split("_")[1] for line in f.read().split("\n") if line])
+        logging.info(f"Already downloaded languages: {langs_to_skip}")
+    else:
+        langs_to_skip = set()
+
+    dl_config = DownloadConfig(
+        cache_dir=root_dir / "cache",
+        resume_download=True,
+        max_retries=5,
+    )
+    dl_manager = DownloadManager(
+        download_config=dl_config,
+        record_checksums=False,
+    )
+
+    for lang_id, lang in enumerate(tqdm(languages, desc="Processing languages...")):
+        if lang in langs_to_skip:
+            logging.info(f"Data for '{lang.upper()}' language already downloaded, skipping it. ")
+            continue
+        try:
+            download_language(dl_manager, lang, root_dir=root_dir)
+            with open(root_dir / "langs_ok.txt", "a") as f:
+                f.write(f"{lang_id}_{lang}\n")
+        except ConnectionError as e:
+            logging.error(e.strerror)
+            with open(root_dir / "langs_failed.txt", "a") as f:
+                f.write(f"{lang_id}_{lang}\n")
+        time.sleep(10)
+
+
+if __name__ == "__main__":
+    main()

langs.json

@@ -0,0 +1,702 @@
+{
+    "ab": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ar": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "as": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ast": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 1,
+        "invalidated": 0
+    },
+    "az": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ba": {
+        "train": 3,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "bas": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "be": {
+        "train": 9,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "bg": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "bn": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 6,
+        "invalidated": 1
+    },
+    "br": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ca": {
+        "train": 23,
+        "dev": 1,
+        "test": 1,
+        "other": 13,
+        "invalidated": 2
+    },
+    "ckb": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "cnh": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "cs": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "cv": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "cy": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "da": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "de": {
+        "train": 12,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 2
+    },
+    "dv": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "el": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "en": {
+        "train": 24,
+        "dev": 1,
+        "test": 1,
+        "other": 8,
+        "invalidated": 7
+    },
+    "eo": {
+        "train": 4,
+        "dev": 1,
+        "test": 1,
+        "other": 4,
+        "invalidated": 4
+    },
+    "es": {
+        "train": 6,
+        "dev": 1,
+        "test": 1,
+        "other": 30,
+        "invalidated": 2
+    },
+    "et": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "eu": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "fa": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "fi": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "fr": {
+        "train": 13,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 2
+    },
+    "fy-NL": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 2,
+        "invalidated": 1
+    },
+    "ga-IE": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "gl": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "gn": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ha": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "hi": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "hsb": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 0,
+        "invalidated": 1
+    },
+    "hu": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "hy-AM": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ia": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "id": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ig": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "it": {
+        "train": 4,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ja": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ka": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "kab": {
+        "train": 4,
+        "dev": 1,
+        "test": 1,
+        "other": 3,
+        "invalidated": 1
+    },
+    "kk": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 0,
+        "invalidated": 1
+    },
+    "kmr": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ky": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "lg": {
+        "train": 2,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "lt": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "lv": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mdf": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mhr": {
+        "train": 2,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mk": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ml": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mn": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mr": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mrj": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "mt": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "myv": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "nan-tw": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ne-NP": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "nl": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "nn-NO": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 0,
+        "invalidated": 1
+    },
+    "or": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "pa-IN": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "pl": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "pt": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "rm-sursilv": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "rm-vallader": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ro": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ru": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "rw": {
+        "train": 26,
+        "dev": 1,
+        "test": 1,
+        "other": 2,
+        "invalidated": 6
+    },
+    "sah": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sat": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sc": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sk": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "skr": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sl": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sr": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sv-SE": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "sw": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 7,
+        "invalidated": 2
+    },
+    "ta": {
+        "train": 2,
+        "dev": 1,
+        "test": 1,
+        "other": 3,
+        "invalidated": 1
+    },
+    "th": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 5,
+        "invalidated": 1
+    },
+    "ti": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 1,
+        "invalidated": 0
+    },
+    "tig": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 0,
+        "invalidated": 1
+    },
+    "tok": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "tr": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "tt": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "tw": {
+        "train": 1,
+        "dev": 0,
+        "test": 0,
+        "other": 0,
+        "invalidated": 0
+    },
+    "ug": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "uk": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "ur": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 3,
+        "invalidated": 1
+    },
+    "uz": {
+        "train": 2,
+        "dev": 1,
+        "test": 1,
+        "other": 4,
+        "invalidated": 1
+    },
+    "vi": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "vot": {
+        "train": 1,
+        "dev": 0,
+        "test": 1,
+        "other": 0,
+        "invalidated": 1
+    },
+    "yue": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "zh-CN": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 18,
+        "invalidated": 1
+    },
+    "zh-HK": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 1,
+        "invalidated": 1
+    },
+    "zh-TW": {
+        "train": 1,
+        "dev": 1,
+        "test": 1,
+        "other": 2,
+        "invalidated": 1
+    }
+}