Repository: mne-tools/mne-bids-pipeline

Issue:
Optimize ds000248 downloading & caching
Currently we download parts of `ds000248` repeatedly for different tests; they end up in the same cache but are placed in separate directories. This can surely be optimized to speed up CI runs quite a bit.
Optimize ds000248 downloading & caching
Currently we download parts of `ds000248` repeatedly for different tests; they end up in the same cache but are placed in separate directories. This can surely be optimized to speed up CI runs quite a bit.


Discussion:

Resolving patch:
diff --git a/config.py b/config.py
--- a/config.py
+++ b/config.py
@@ -2976,24 +2976,3 @@ def save_logs(logs):
            'configuration. Currently the `conditions` parameter is empty. '
            'This is only allowed for resting-state analysis.')
     raise ValueError(msg)
-
-
-# # Leave this here for reference for now
-#
-# _preproc_funcs_path = (pathlib.Path(__file__).parent / 'scripts' /
-#                        'preprocessing' / 'common_functions.py')
-# _preproc_funcs = runpy.run_path(_preproc_funcs_path)
-# _preproc_funcs = {
-#     func_name: func
-#     for func_name, func in _preproc_funcs.items()
-#     if func in _preproc_funcs['exports']
-# }
-#
-#
-# class Funcs(TypedDict):
-#     preprocessing: Dict[str, Callable]
-#
-#
-# funcs = Funcs(
-#     preprocessing=_preproc_funcs
-# )
diff --git a/docs/source/examples/gen_examples.py b/docs/source/examples/gen_examples.py
--- a/docs/source/examples/gen_examples.py
+++ b/docs/source/examples/gen_examples.py
@@ -67,8 +67,8 @@ def _bool_to_icon(x: Union[bool, Iterable]) -> str:
     example_target_dir = Path(f'docs/source/examples/{dataset_name}')
     example_target_dir.mkdir(exist_ok=True)
 
-    example_source_dir = Path(f'~/mne_data/{dataset_name}'
-                              '/derivatives/mne-bids-pipeline').expanduser()
+    example_source_dir = Path(f'~/mne_data/derivatives/mne-bids-pipeline/'
+                              f'{dataset_name}').expanduser()
     html_report_fnames = list(example_source_dir.rglob('*.html'))
 
     if not html_report_fnames:
diff --git a/scripts/report/01-make_reports.py b/scripts/report/01-make_reports.py
--- a/scripts/report/01-make_reports.py
+++ b/scripts/report/01-make_reports.py
@@ -695,6 +695,25 @@ def get_config(
     subject: Optional[str] = None,
     session: Optional[str] = None
 ) -> BunchConst:
+    # Deal with configurations where `deriv_root` was specified, but not
+    # `fs_subjects_dir`. We normally raise an exception in this case in
+    # `get_fs_subjects_dir()`. However, in situations where users only run the
+    # sensor-space scripts, we never call this function, so everything works
+    # totally fine at first (which is expected). Yet, when creating the
+    # reports, the pipeline would fail with an exception – which is
+    # unjustified, as it would not make sense to force users to provide an
+    # `fs_subjects_dir` if they don't care about source analysis anyway! So
+    # simply assign a dummy value in such cases.
+    # `get_fs_subject()` calls `get_fs_subjects_dir()`, so take care of this
+    # too.
+    try:
+        fs_subjects_dir = config.get_fs_subjects_dir()
+    except ValueError:
+        fs_subjects_dir = None
+        fs_subject = None
+    else:
+        fs_subject = config.get_fs_subject(subject=subject)
+
     cfg = BunchConst(
         task=config.get_task(),
         runs=config.get_runs(subject=subject),
@@ -715,8 +734,8 @@ def get_config(
         decoding_metric=config.decoding_metric,
         n_boot=config.n_boot,
         inverse_method=config.inverse_method,
-        fs_subject=config.get_fs_subject(subject=subject),
-        fs_subjects_dir=config.get_fs_subjects_dir(),
+        fs_subject=fs_subject,
+        fs_subjects_dir=fs_subjects_dir,
         deriv_root=config.get_deriv_root(),
         bids_root=config.get_bids_root(),
         use_template_mri=config.use_template_mri,


Test patch:
diff --git a/tests/configs/config_ERP_CORE.py b/tests/configs/config_ERP_CORE.py
--- a/tests/configs/config_ERP_CORE.py
+++ b/tests/configs/config_ERP_CORE.py
@@ -28,6 +28,7 @@
 
 study_name = 'ERP-CORE'
 bids_root = '~/mne_data/ERP_CORE'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ERP_CORE'
 
 task = os.environ.get('MNE_BIDS_STUDY_TASK')
 sessions = [task]
diff --git a/tests/configs/config_ds000117.py b/tests/configs/config_ds000117.py
--- a/tests/configs/config_ds000117.py
+++ b/tests/configs/config_ds000117.py
@@ -4,6 +4,8 @@
 
 study_name = 'ds000117'
 bids_root = '~/mne_data/ds000117'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000117'
+
 task = 'facerecognition'
 ch_types = ['meg']
 runs = ['01', '02']
diff --git a/tests/configs/config_ds000246.py b/tests/configs/config_ds000246.py
--- a/tests/configs/config_ds000246.py
+++ b/tests/configs/config_ds000246.py
@@ -4,8 +4,9 @@
 
 study_name = 'ds000246'
 bids_root = '~/mne_data/ds000246'
-deriv_root = '~/mne_data/ds000246/derivatives/mne-bids-pipeline'
-subjects_dir = '~/mne_data/ds000246/derivatives/freesurfer/subjects'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000246'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
+
 runs = ['01']
 l_freq = .3
 h_freq = 100.
diff --git a/tests/configs/config_ds000247.py b/tests/configs/config_ds000247.py
--- a/tests/configs/config_ds000247.py
+++ b/tests/configs/config_ds000247.py
@@ -4,6 +4,7 @@
 
 study_name = 'ds000247'
 bids_root = '~/mne_data/ds000247'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000247'
 
 subjects = ['0002']
 sessions = ['01']
diff --git a/tests/configs/config_ds000248.py b/tests/configs/config_ds000248.py
--- a/tests/configs/config_ds000248.py
+++ b/tests/configs/config_ds000248.py
@@ -4,6 +4,8 @@
 
 study_name = 'ds000248'
 bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
 
 subjects = ['01']
 rename_events = {'Smiley': 'Emoji',
diff --git a/tests/configs/config_ds000248_FLASH_BEM.py b/tests/configs/config_ds000248_FLASH_BEM.py
--- a/tests/configs/config_ds000248_FLASH_BEM.py
+++ b/tests/configs/config_ds000248_FLASH_BEM.py
@@ -2,7 +2,10 @@
 MNE Sample Data: BEM fromm FLASH images
 """
 study_name = 'ds000248'
-bids_root = '~/mne_data/ds000248_FLASH_BEM'
+bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248_FLASH_BEM'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
+
 subjects = ['01']
 conditions = ['Auditory']
 
diff --git a/tests/configs/config_ds000248_T1_BEM.py b/tests/configs/config_ds000248_T1_BEM.py
--- a/tests/configs/config_ds000248_T1_BEM.py
+++ b/tests/configs/config_ds000248_T1_BEM.py
@@ -3,7 +3,10 @@
 """
 
 study_name = 'ds000248'
-bids_root = '~/mne_data/ds000248_T1_BEM'
+bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248_T1_BEM'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
+
 subjects = ['01']
 conditions = ['Auditory']
 
diff --git a/tests/configs/config_ds000248_coreg_surfaces.py b/tests/configs/config_ds000248_coreg_surfaces.py
--- a/tests/configs/config_ds000248_coreg_surfaces.py
+++ b/tests/configs/config_ds000248_coreg_surfaces.py
@@ -3,7 +3,10 @@
 """
 
 study_name = 'ds000248'
-bids_root = '~/mne_data/ds000248_coreg_surfaces'
+bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248_coreg_surfaces'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
+
 subjects = ['01']
 conditions = ['Auditory']
 ch_types = ['meg']
diff --git a/tests/configs/config_ds000248_ica.py b/tests/configs/config_ds000248_ica.py
--- a/tests/configs/config_ds000248_ica.py
+++ b/tests/configs/config_ds000248_ica.py
@@ -2,7 +2,9 @@
 MNE Sample Data: ICA
 """
 study_name = 'MNE "sample" dataset'
-bids_root = '~/mne_data/ds000248_ica'
+bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248_ica'
+
 ch_types = ['meg']
 data_type = 'meg'
 
diff --git a/tests/configs/config_ds000248_no_mri.py b/tests/configs/config_ds000248_no_mri.py
--- a/tests/configs/config_ds000248_no_mri.py
+++ b/tests/configs/config_ds000248_no_mri.py
@@ -3,7 +3,9 @@
 """
 
 study_name = 'ds000248'
-bids_root = '~/mne_data/ds000248_no_mri'
+bids_root = '~/mne_data/ds000248'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000248_no_mri'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
 
 subjects = ['01']
 rename_events = {'Smiley': 'Emoji',
diff --git a/tests/configs/config_ds001810.py b/tests/configs/config_ds001810.py
--- a/tests/configs/config_ds001810.py
+++ b/tests/configs/config_ds001810.py
@@ -4,6 +4,7 @@
 
 study_name = 'ds001810'
 bids_root = '~/mne_data/ds001810'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds001810'
 
 task = 'attentionalblink'
 interactive = False
diff --git a/tests/configs/config_ds001971.py b/tests/configs/config_ds001971.py
--- a/tests/configs/config_ds001971.py
+++ b/tests/configs/config_ds001971.py
@@ -14,6 +14,7 @@
 
 study_name = 'ds001971'
 bids_root = '~/mne_data/ds001971'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds001971'
 
 task = 'AudioCueWalkingStudy'
 interactive = False
diff --git a/tests/configs/config_ds003104.py b/tests/configs/config_ds003104.py
--- a/tests/configs/config_ds003104.py
+++ b/tests/configs/config_ds003104.py
@@ -2,6 +2,8 @@
 """
 study_name = 'MNE-somato-data-anonymized'
 bids_root = '~/mne_data/ds003104'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds003104'
+subjects_dir = f'{bids_root}/derivatives/freesurfer/subjects'
 
 conditions = ['somato_event1']
 ch_types = ['meg']
diff --git a/tests/configs/config_ds003392.py b/tests/configs/config_ds003392.py
--- a/tests/configs/config_ds003392.py
+++ b/tests/configs/config_ds003392.py
@@ -3,6 +3,7 @@
 """
 study_name = 'localizer'
 bids_root = '~/mne_data/ds003392'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds003392'
 
 subjects = ['01']
 
diff --git a/tests/configs/config_eeg_matchingpennies.py b/tests/configs/config_eeg_matchingpennies.py
--- a/tests/configs/config_eeg_matchingpennies.py
+++ b/tests/configs/config_eeg_matchingpennies.py
@@ -4,6 +4,7 @@
 
 study_name = 'eeg_matchingpennies'
 bids_root = '~/mne_data/eeg_matchingpennies'
+deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/eeg_matchingpennies'
 
 subjects = ['05']
 task = 'matchingpennies'
diff --git a/tests/datasets.py b/tests/datasets.py
--- a/tests/datasets.py
+++ b/tests/datasets.py
@@ -70,8 +70,11 @@ class DATASET_OPTIONS_T(TypedDict):
         'openneuro': 'ds000248',
         'osf': '',
         'web': '',
-        'include': ['sub-01', 'sub-emptyroom',
-                    'derivatives/freesurfer/subjects'],
+        'include': [
+            'sub-01',
+            'sub-emptyroom',
+            'derivatives/freesurfer/subjects'
+        ],
         'exclude': [
             'derivatives/freesurfer/subjects/fsaverage/mri/aparc.a2005s+aseg.mgz',  # noqa: E501
             'derivatives/freesurfer/subjects/fsaverage/mri/aparc+aseg.mgz',
@@ -83,61 +86,6 @@ class DATASET_OPTIONS_T(TypedDict):
             'derivatives/freesurfer/subjects/sub-01/mri/aparc.a2009s+aseg.mgz'
         ],
     },
-    'ds000248_ica': {
-        'git': '',
-        'openneuro': 'ds000248',
-        'osf': '',
-        'web': '',
-        'include': ['sub-01'],
-        'exclude': []
-    },
-    'ds000248_T1_BEM': {
-        'git': '',
-        'openneuro': 'ds000248',
-        'osf': '',
-        'web': '',
-        'include': ['derivatives/freesurfer/subjects/sub-01'],
-        'exclude': [
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc+aseg.mgz',
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.a2009s+aseg.mgz'
-        ],
-    },
-    'ds000248_FLASH_BEM': {
-        'git': '',
-        'openneuro': 'ds000248',
-        'osf': '',
-        'web': '',
-        'include': ['derivatives/freesurfer/subjects/sub-01'],
-        'exclude': [
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc+aseg.mgz',
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.a2009s+aseg.mgz'
-        ],
-    },
-    'ds000248_coreg_surfaces': {
-        'git': '',
-        'openneuro': 'ds000248',
-        'osf': '',
-        'web': '',
-        'include': ['derivatives/freesurfer/subjects/sub-01'],
-        'exclude': [
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc+aseg.mgz',
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.DKTatlas+aseg.mgz',  # noqa: E501
-            'derivatives/freesurfer/subjects/sub-01/mri/aparc.a2009s+aseg.mgz'
-        ],
-    },
-    'ds000248_no_mri': {
-        'git': '',
-        'openneuro': 'ds000248',
-        'osf': '',
-        'web': '',
-        'include': ['sub-01'],
-        'exclude': ['sub-01/anat'],
-    },
     'ds000117': {
         'git': '',
         'openneuro': 'ds000117',
diff --git a/tests/run_tests.py b/tests/run_tests.py
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -67,35 +67,35 @@ class TestOptionsT(TypedDict):
         'env': {}
     },
     'ds000248_ica': {
-        'dataset': 'ds000248_ica',
+        'dataset': 'ds000248',
         'config': 'config_ds000248_ica.py',
         'steps': ('preprocessing', 'sensor', 'report'),
         'task': None,
         'env': {}
     },
     'ds000248_T1_BEM': {
-        'dataset': 'ds000248_T1_BEM',
+        'dataset': 'ds000248',
         'config': 'config_ds000248_T1_BEM.py',
         'steps': ('source/make_bem_surfaces',),
         'task': None,
         'env': {}
     },
     'ds000248_FLASH_BEM': {
-        'dataset': 'ds000248_FLASH_BEM',
+        'dataset': 'ds000248',
         'config': 'config_ds000248_FLASH_BEM.py',
         'steps': ('source/make_bem_surfaces',),
         'task': None,
         'env': {}
     },
     'ds000248_coreg_surfaces': {
-        'dataset': 'ds000248_coreg_surfaces',
+        'dataset': 'ds000248',
         'config': 'config_ds000248_coreg_surfaces.py',
         'steps': ('freesurfer/coreg_surfaces',),
         'task': None,
         'env': {}
     },
     'ds000248_no_mri': {
-        'dataset': 'ds000248_no_mri',
+        'dataset': 'ds000248',
         'config': 'config_ds000248_no_mri.py',
         'steps': ('preprocessing', 'sensor', 'source', 'report'),
         'task': None,
@@ -269,12 +269,13 @@ def run_tests(test_suite, download):
         if os.environ.get('DATASET') is None:
             parser.print_help()
         print('\n\n')
-        raise KeyError('"{}" is not a valid dataset key in the TEST_SUITE '
-                       'dictionary in the run_tests.py module. Use one of {}.'
-                       .format(args.dataset, ', '.join(TEST_SUITE.keys())))
+        raise KeyError(
+            f'"{args.dataset}" is not a valid dataset key in the TEST_SUITE '
+            f'dictionary in the run_tests.py module. Use one of: '
+            f'{", ".join(TEST_SUITE.keys())}.'
+        )
 
     # Run the tests
-    print('Running the following tests: {}'
-          .format(', '.join(test_suite.keys())))
+    print(f'Running the following tests: {", ".join(test_suite.keys())}')
 
     run_tests(test_suite, download=download)