Repository: mne-tools/mne-bids-pipeline

Issue:
MAINT: Use pooch for web downloads
@hoechenberger any reason you didn't use pooch for the ~50 lines in `_download.py`? Seems like we would get some minus lines plus more robustness/maintained-ness by using `pooch`.


Discussion:
I don't remember. If you think it can be replaced using pooch, please do go for it!

Resolving patch:
diff --git a/docs/source/examples/gen_examples.py b/docs/source/examples/gen_examples.py
--- a/docs/source/examples/gen_examples.py
+++ b/docs/source/examples/gen_examples.py
@@ -142,7 +142,7 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
         logger.warning(f"Dataset {dataset_name} has no HTML report.")
         continue
 
-    options = DATASET_OPTIONS[dataset_options_key]
+    options = DATASET_OPTIONS[dataset_options_key].copy()  # we modify locally
 
     report_str = "\n## Generated output\n\n"
     example_target_dir = this_dir / dataset_name
@@ -198,20 +198,24 @@ def _gen_demonstrated_funcs(example_config_path: Path) -> dict:
             f"{fname.name} :fontawesome-solid-square-poll-vertical:</a>\n\n"
         )
 
-    if options["openneuro"]:
+    assert sum(key in options for key in ("openneuro", "git", "web", "datalad")) == 1
+    if "openneuro" in options:
         url = f'https://openneuro.org/datasets/{options["openneuro"]}'
-    elif options["git"]:
+    elif "git" in options:
         url = options["git"]
-    elif options["web"]:
+    elif "web" in options:
         url = options["web"]
     else:
+        assert "datalad" in options  # guaranteed above
         url = ""
 
     source_str = (
         f"## Dataset source\n\nThis dataset was acquired from " f"[{url}]({url})\n"
     )
 
-    if options["openneuro"]:
+    if "openneuro" in options:
+        for key in ("include", "exclude"):
+            options[key] = options.get(key, [])
         download_str = (
             f'\n??? example "How to download this dataset"\n'
             f"    Run in your terminal:\n"
diff --git a/mne_bids_pipeline/_download.py b/mne_bids_pipeline/_download.py
--- a/mne_bids_pipeline/_download.py
+++ b/mne_bids_pipeline/_download.py
@@ -13,7 +13,10 @@ def _download_via_datalad(*, ds_name: str, ds_path: Path):
     import datalad.api as dl
 
     print('datalad installing "{}"'.format(ds_name))
-    git_url = DATASET_OPTIONS[ds_name]["git"]
+    options = DATASET_OPTIONS[ds_name]
+    git_url = options["git"]
+    assert "exclude" not in options
+    assert "hash" not in options
     dataset = dl.install(path=ds_path, source=git_url)
 
     # XXX: git-annex bug:
@@ -24,7 +27,7 @@ def _download_via_datalad(*, ds_name: str, ds_path: Path):
     else:
         n_jobs = 1
 
-    for to_get in DATASET_OPTIONS[ds_name]["include"]:
+    for to_get in DATASET_OPTIONS[ds_name].get("include", []):
         print('datalad get data "{}" for "{}"'.format(to_get, ds_name))
         dataset.get(to_get, jobs=n_jobs)
 
@@ -32,23 +35,27 @@ def _download_via_datalad(*, ds_name: str, ds_path: Path):
 def _download_via_openneuro(*, ds_name: str, ds_path: Path):
     import openneuro
 
+    options = DATASET_OPTIONS[ds_name]
+    assert "hash" not in options
+
     openneuro.download(
-        dataset=DATASET_OPTIONS[ds_name]["openneuro"],
+        dataset=options["openneuro"],
         target_dir=ds_path,
-        include=DATASET_OPTIONS[ds_name]["include"],
-        exclude=DATASET_OPTIONS[ds_name]["exclude"],
+        include=options.get("include", []),
+        exclude=options.get("exclude", []),
         verify_size=False,
     )
 
 
 def _download_from_web(*, ds_name: str, ds_path: Path):
     """Retrieve Zip archives from a web URL."""
-    import cgi
-    import zipfile
-    import httpx
-    from tqdm import tqdm
+    import pooch
 
-    url = DATASET_OPTIONS[ds_name]["web"]
+    options = DATASET_OPTIONS[ds_name]
+    url = options["web"]
+    known_hash = options["hash"]
+    assert "exclude" not in options
+    assert "include" not in options
     if ds_path.exists():
         print(
             "Dataset directory already exists; remove it if you wish to "
@@ -57,65 +64,26 @@ def _download_from_web(*, ds_name: str, ds_path: Path):
         return
 
     ds_path.mkdir(parents=True, exist_ok=True)
-
-    with httpx.Client(follow_redirects=True) as client:
-        with client.stream("GET", url=url) as response:
-            if not response.is_error:
-                pass  # All good!
-            else:
-                raise RuntimeError(
-                    f"Error {response.status_code} when trying " f"to download {url}"
-                )
-
-            header = response.headers["content-disposition"]
-            _, params = cgi.parse_header(header)
-            # where to store the archive
-            outfile = ds_path / params["filename"]
-            remote_file_size = int(response.headers["content-length"])
-
-            with open(outfile, mode="wb") as f:
-                with tqdm(
-                    desc=params["filename"],
-                    initial=0,
-                    total=remote_file_size,
-                    unit="B",
-                    unit_scale=True,
-                    unit_divisor=1024,
-                    leave=False,
-                ) as progress:
-                    num_bytes_downloaded = response.num_bytes_downloaded
-
-                    for chunk in response.iter_bytes():
-                        f.write(chunk)
-                        progress.update(
-                            response.num_bytes_downloaded - num_bytes_downloaded
-                        )
-                        num_bytes_downloaded = response.num_bytes_downloaded
-
-        assert outfile.suffix == ".zip"
-
-        with zipfile.ZipFile(outfile) as zip:
-            for zip_info in zip.infolist():
-                path_in_zip = Path(zip_info.filename)
-                # omit top-level directory from Zip archive
-                target_path = str(Path(*path_in_zip.parts[1:]))
-                if str(target_path) in (".", ".."):
-                    continue
-                if zip_info.filename.endswith("/"):
-                    (ds_path / target_path).mkdir(parents=True, exist_ok=True)
-                    continue
-                zip_info.filename = target_path
-                print(f"Extracting: {target_path}")
-                zip.extract(zip_info, ds_path)
-
-        outfile.unlink()
+    path = ds_path.parent.resolve(strict=True)
+    fname = f"{ds_name}.zip"
+    pooch.retrieve(
+        url=url,
+        path=path,
+        fname=fname,
+        processor=pooch.Unzip(extract_dir="."),  # relative to path
+        progressbar=True,
+        known_hash=known_hash,
+    )
+    (path / f"{ds_name}.zip").unlink()
 
 
 def _download(*, ds_name: str, ds_path: Path):
-    openneuro_name = DATASET_OPTIONS[ds_name]["openneuro"]
-    git_url = DATASET_OPTIONS[ds_name]["git"]
-    osf_node = DATASET_OPTIONS[ds_name]["osf"]
-    web_url = DATASET_OPTIONS[ds_name]["web"]
+    options = DATASET_OPTIONS[ds_name]
+    openneuro_name = options.get("openneuro", "")
+    git_url = options.get("git", "")
+    osf_node = options.get("osf", "")
+    web_url = options.get("web", "")
+    assert sum(bool(x) for x in (openneuro_name, git_url, osf_node, web_url)) == 1
 
     if openneuro_name:
         download_func = _download_via_openneuro
@@ -123,10 +91,9 @@ def _download(*, ds_name: str, ds_path: Path):
         download_func = _download_via_datalad
     elif osf_node:
         raise RuntimeError("OSF downloads are currently not supported.")
-    elif web_url:
-        download_func = _download_from_web
     else:
-        raise ValueError("No download location was specified.")
+        assert web_url
+        download_func = _download_from_web
 
     download_func(ds_name=ds_name, ds_path=ds_path)
 


Test patch:
diff --git a/mne_bids_pipeline/tests/datasets.py b/mne_bids_pipeline/tests/datasets.py
--- a/mne_bids_pipeline/tests/datasets.py
+++ b/mne_bids_pipeline/tests/datasets.py
@@ -3,23 +3,22 @@
 from typing import Dict, List, TypedDict
 
 
-class DATASET_OPTIONS_T(TypedDict):
-    git: str
-    openneuro: str
-    osf: str
-    web: str
-    include: List[str]
-    exclude: List[str]
+# If not supplied below, the effective defaults are listed in comments
+class DATASET_OPTIONS_T(TypedDict, total=False):
+    git: str  # ""
+    openneuro: str  # ""
+    osf: str  # ""
+    web: str  # ""
+    include: List[str]  # []
+    exclude: List[str]  # []
+    hash: str  # ""
 
 
 DATASET_OPTIONS: Dict[str, DATASET_OPTIONS_T] = {
     "ERP_CORE": {
-        "git": "",
-        "openneuro": "",
-        "osf": "",  # original dataset: '9f5w7'
+        # original dataset: "osf": "9f5w7"
         "web": "https://osf.io/3zk6n/download?version=2",
-        "include": [],
-        "exclude": [],
+        "hash": "sha256:ddc94a7c9ba1922637f2770592dd51c019d341bf6bc8558e663e1979a4cb002f",  # noqa: E501
     },
     "eeg_matchingpennies": {
         # This dataset started out on osf.io as dataset https://osf.io/cj2dr
@@ -31,18 +30,12 @@ class DATASET_OPTIONS_T(TypedDict):
         #   "include": ["sub-05"],
         #
         # So now we mirror this datalad-fetched git repo back on osf.io!
-        "git": "",
-        "openneuro": "",
-        "osf": "",  # original dataset: 'cj2dr'
+        # original dataset: "osf": "cj2dr"
         "web": "https://osf.io/download/8rbfk?version=1",
-        "include": [],
-        "exclude": [],
+        "hash": "sha256:06bfbe52c50b9343b6b8d2a5de3dd33e66ad9303f7f6bfbe6868c3c7c375fafd",  # noqa: E501
     },
     "ds003104": {  # Anonymized "somato" dataset.
-        "git": "",
         "openneuro": "ds003104",
-        "osf": "",
-        "web": "",
         "include": ["sub-01", "derivatives/freesurfer/subjects"],
         "exclude": [
             "derivatives/freesurfer/subjects/01/mri/aparc+aseg.mgz",
@@ -51,30 +44,19 @@ class DATASET_OPTIONS_T(TypedDict):
         ],
     },
     "ds000246": {
-        "git": "",
         "openneuro": "ds000246",
-        "osf": "",
-        "web": "",
         "include": [
             "sub-0001/meg/sub-0001_task-AEF_run-01_meg.ds",
             "sub-0001/meg/sub-0001_task-AEF_run-01_meg.json",
             "sub-0001/meg/sub-0001_task-AEF_run-01_channels.tsv",
         ],
-        "exclude": [],
     },
     "ds000247": {
-        "git": "",
         "openneuro": "ds000247",
-        "osf": "",
-        "web": "",
         "include": ["sub-0002/ses-01/meg"],
-        "exclude": [],
     },
     "ds000248": {
-        "git": "",
         "openneuro": "ds000248",
-        "osf": "",
-        "web": "",
         "include": ["sub-01", "sub-emptyroom", "derivatives/freesurfer/subjects"],
         "exclude": [
             "derivatives/freesurfer/subjects/fsaverage/mri/aparc.a2005s+aseg.mgz",  # noqa: E501
@@ -88,10 +70,7 @@ class DATASET_OPTIONS_T(TypedDict):
         ],
     },
     "ds000117": {
-        "git": "",
         "openneuro": "ds000117",
-        "osf": "",
-        "web": "",
         "include": [
             "sub-01/ses-meg/meg/sub-01_ses-meg_task-facerecognition_run-01_*",  # noqa: E501
             "sub-01/ses-meg/meg/sub-01_ses-meg_task-facerecognition_run-02_*",  # noqa: E501
@@ -102,29 +81,17 @@ class DATASET_OPTIONS_T(TypedDict):
             "derivatives/meg_derivatives/ct_sparse.fif",
             "derivatives/meg_derivatives/sss_cal.dat",
         ],
-        "exclude": [],
     },
     "ds003775": {
-        "git": "",
         "openneuro": "ds003775",
-        "osf": "",
-        "web": "",
         "include": ["sub-010"],
-        "exclude": [],
     },
     "ds001810": {
-        "git": "",
         "openneuro": "ds001810",
-        "osf": "",
-        "web": "",
         "include": ["sub-01"],
-        "exclude": [],
     },
     "ds001971": {
-        "git": "",
         "openneuro": "ds001971",
-        "osf": "",
-        "web": "",
         "include": [
             "sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_events.tsv",
             "sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_eeg.set",
@@ -134,38 +101,25 @@ class DATASET_OPTIONS_T(TypedDict):
             "sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_coordsystem.json",  # noqa: E501
             "sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_channels.tsv",  # noqa: E501
         ],
-        "exclude": [],
     },
     "ds003392": {
-        "git": "",
         "openneuro": "ds003392",
-        "osf": "",
-        "web": "",
         "include": ["sub-01", "sub-emptyroom/ses-19111211"],
-        "exclude": [],
     },
     "ds004107": {
-        "git": "",
         "openneuro": "ds004107",
-        "osf": "",
-        "web": "",
         "include": [
             "sub-mind002/ses-01/meg/*coordsystem*",
             "sub-mind002/ses-01/meg/*auditory*",
         ],
-        "exclude": [],
     },
     "ds004229": {
-        "git": "",
         "openneuro": "ds004229",
-        "osf": "",
-        "web": "",
         "include": [
             "sub-102",
             "sub-emptyroom/ses-20000101",
             "derivatives/meg_derivatives/ct_sparse.fif",
             "derivatives/meg_derivatives/sss_cal.dat",
         ],
-        "exclude": [],
     },
 }
diff --git a/mne_bids_pipeline/tests/test_run.py b/mne_bids_pipeline/tests/test_run.py
--- a/mne_bids_pipeline/tests/test_run.py
+++ b/mne_bids_pipeline/tests/test_run.py
@@ -18,26 +18,17 @@
 
 
 # Once PEP655 lands in 3.11 we can use NotRequired instead of total=False
+# Effective defaults are listed in comments
 class _TestOptionsT(TypedDict, total=False):
-    dataset: str
-    config: str
-    steps: Collection[str]
-    task: Optional[str]
-    env: Dict[str, str]
-    requires: Collection[str]
+    dataset: str  # key.split("_")[0]
+    config: str  # f"config_{key}.py"
+    steps: Collection[str]  # ("preprocessing", "sensor")
+    task: Optional[str]  # None
+    env: Dict[str, str]  # {}
+    requires: Collection[str]  # ()
+    extra_config: str  # ""
 
 
-# If not supplied below, the defaults are:
-# key: {
-#     "dataset": key.split("_")[0],
-#     "config": f"config_{key}.py",
-#     "steps": ("preprocessing", "sensor"),
-#     "env": {},
-#     "task": None,
-#     "requires": (),
-#     "extra_config": "",
-# }
-
 TEST_SUITE: Dict[str, _TestOptionsT] = {
     "ds003392": {},
     "ds004229": {},