Repository: mne-tools/mne-bids-pipeline

Issue:
add ds004107 to datasets
we should add the dataset to our examples:

https://openneuro.org/datasets/ds004107/




Discussion:
Hello! 👋 Thanks for opening your first issue here! ❤️ We will try to get back to you soon. 🚴🏽‍♂️

@agramfort what is the advantage / difference compared to what we already have?
It’s a super clean MEG dataset and for now the more datasets we add the better  to make our point with the pipeline 


Resolving patch:
diff --git a/docs/source/examples/gen_examples.py b/docs/source/examples/gen_examples.py
--- a/docs/source/examples/gen_examples.py
+++ b/docs/source/examples/gen_examples.py
@@ -71,6 +71,7 @@ def _bool_to_icon(x: Union[bool, Iterable]) -> str:
         dataset_name = test_dataset_options['dataset']
     else:
         dataset_name = test_name
+    del test_dataset_options
 
     example_target_dir = Path(f'docs/source/examples/{dataset_name}')
     example_target_dir.mkdir(exist_ok=True)
@@ -96,7 +97,9 @@ def _bool_to_icon(x: Union[bool, Iterable]) -> str:
     else:
         dataset_name = test_dataset_name
 
-    del test_dataset_name
+    dataset_options_key = test_dataset_options.get(
+        'dataset', test_dataset_name.split('_')[0])
+    del test_dataset_options, test_dataset_name
 
     if dataset_name in datasets_without_html:
         logger.warning(f'Dataset {dataset_name} has no HTML report.')
@@ -104,7 +107,7 @@ def _bool_to_icon(x: Union[bool, Iterable]) -> str:
 
     logger.warning(f'Generating markdown file for dataset: {dataset_name}')
 
-    options = dataset_options[test_dataset_options['dataset']]
+    options = dataset_options[dataset_options_key]
 
     report_str = '\n## Generated output\n\n'
     example_target_dir = Path(f'docs/source/examples/{dataset_name}')


Test patch:
diff --git a/tests/configs/config_ds000247.py b/tests/configs/config_ds000247.py
--- a/tests/configs/config_ds000247.py
+++ b/tests/configs/config_ds000247.py
@@ -5,8 +5,8 @@
 
 
 study_name = 'ds000247'
-bids_root = '~/mne_data/ds000247'
-deriv_root = '~/mne_data/derivatives/mne-bids-pipeline/ds000247'
+bids_root = f'~/mne_data/{study_name}'
+deriv_root = f'~/mne_data/derivatives/mne-bids-pipeline/{study_name}'
 
 subjects = ['0002']
 sessions = ['01']
diff --git a/tests/configs/config_ds001971.py b/tests/configs/config_ds001971.py
--- a/tests/configs/config_ds001971.py
+++ b/tests/configs/config_ds001971.py
@@ -20,9 +20,7 @@
 interactive = False
 ch_types = ['eeg']
 reject = {'eeg': 150e-6}
-conditions = ['left', 'right']
-contrasts = [('left', 'right')]
-decode = True
+conditions = ['AdvanceTempo', 'DelayTempo']
 
 subjects = ['001']
 runs = ['01']
diff --git a/tests/configs/config_ds004107.py b/tests/configs/config_ds004107.py
new file mode 100644
--- /dev/null
+++ b/tests/configs/config_ds004107.py
@@ -0,0 +1,23 @@
+"""
+MIND DATA
+
+M.P. Weisend, F.M. Hanlon, R. Montaño, S.P. Ahlfors, A.C. Leuthold,
+D. Pantazis, J.C. Mosher, A.P. Georgopoulos, M.S. Hämäläinen, C.J.
+Aine,, V. (2007).
+Paving the way for cross-site pooling of magnetoencephalography (MEG) data.
+International Congress Series, Volume 1300, Pages 615-618.
+"""
+# This has auditory, median, indx, visual, rest, and emptyroom but let's just
+# process the auditory (it's the smallest after rest)
+study_name = 'ds004107'
+bids_root = f'~/mne_data/{study_name}'
+deriv_root = f'~/mne_data/derivatives/mne-bids-pipeline/{study_name}'
+subjects = ['mind002']
+sessions = ['01']
+conditions = ['left', 'right']  # there are also tone and noise
+task = 'auditory'
+ch_types = ['meg']
+crop_runs = (0, 100)  # to speed up computations
+spatial_filter = 'ssp'
+l_freq = 1.0
+h_freq = 40.0
diff --git a/tests/datasets.py b/tests/datasets.py
--- a/tests/datasets.py
+++ b/tests/datasets.py
@@ -129,7 +129,7 @@ class DATASET_OPTIONS_T(TypedDict):
         'osf': '',
         'web': '',
         'include': [
-            'sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_events.tsv'
+            'sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_events.tsv',
             'sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_eeg.set',
             'sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_eeg.fdt',
             'sub-001/eeg/sub-001_task-AudioCueWalkingStudy_run-01_eeg.json',
@@ -150,6 +150,17 @@ class DATASET_OPTIONS_T(TypedDict):
         ],
         'exclude': []
     },
+    'ds004107': {
+        'git': '',
+        'openneuro': 'ds004107',
+        'osf': '',
+        'web': '',
+        'include': [
+            'sub-mind002/ses-01/meg/*coordsystem*',
+            'sub-mind002/ses-01/meg/*auditory*',
+        ],
+        'exclude': []
+    },
     'ds004229': {
         'git': '',
         'openneuro': 'ds004229',
diff --git a/tests/ds001971_participants.tsv b/tests/ds001971_participants.tsv
new file mode 100644
--- /dev/null
+++ b/tests/ds001971_participants.tsv
@@ -0,0 +1,21 @@
+participant_id	sex	age	handedness
+sub-001	F	18-38	R
+sub-002	F	28	R
+sub-003	F	18-38	R
+sub-004	F	18-38	R
+sub-005	M	18-38	R
+sub-006	M	18-38	R
+sub-007	M	32	R
+sub-008	M	38	R
+sub-009	F	18-38	R
+sub-010	F	23	R
+sub-011	M	18-38	R
+sub-012	F	18-38	R
+sub-013	M	18-38	R
+sub-014	M	18-38	R
+sub-015	M	28	R
+sub-016	M	18-38	R
+sub-017	M	18-38	R
+sub-018	F	18-38	R
+sub-019	F	18-38	R
+sub-020	M	18-38	R
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -12,3 +12,4 @@ openneuro-py
 httpx >= 0.20
 tqdm
 Pygments <= 2.11.2
+pyyaml
diff --git a/tests/run_tests.py b/tests/run_tests.py
--- a/tests/run_tests.py
+++ b/tests/run_tests.py
@@ -27,7 +27,8 @@ def fetch(dataset=None):
 DATA_DIR = Path('~/mne_data').expanduser()
 
 
-class TestOptionsT(TypedDict):
+# Once PEP655 lands in 3.11 we can use NotRequired instead of total=False
+class TestOptionsT(TypedDict, total=False):
     dataset: str
     config: str
     steps: Collection[str]
@@ -35,162 +36,89 @@ class TestOptionsT(TypedDict):
     env: Dict[str, str]
 
 
+# If not supplied below, the defaults are:
+# key: {
+#     'dataset': key.split('_')[0],
+#     'config': f'config_{key}.py',
+#     'steps': ('preprocessing', 'sensor', 'report'),
+#     'env': {},
+#     'task': None,
+# }
+#
 TEST_SUITE: Dict[str, TestOptionsT] = {
-    'ds003392': {
-        'dataset': 'ds003392',
-        'config': 'config_ds003392.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
-    },
-    'ds004229': {
-        'dataset': 'ds004229',
-        'config': 'config_ds004229.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
+    'ds003392': {},
+    'ds004229': {},
+    'ds001971': {},
+    'ds004107': {},
+    'ds000117': {},
+    'ds003775': {},
+    'eeg_matchingpennies': {
+        'dataset': 'eeg_matchingpennies',
     },
     'ds000246': {
-        'dataset': 'ds000246',
-        'config': 'config_ds000246.py',
         'steps': ('preprocessing',
                   'preprocessing/make_epochs',  # Test the group/step syntax
                   'sensor', 'report'),
-        'task': None,
-        'env': {}
     },
     'ds000247': {
-        'dataset': 'ds000247',
-        'config': 'config_ds000247.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'rest',
-        'env': {}
     },
     'ds000248': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248.py',
         'steps': ('preprocessing', 'sensor', 'source', 'report'),
-        'task': None,
-        'env': {}
-    },
-    'ds000248_ica': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248_ica.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
     },
+    'ds000248_ica': {},
     'ds000248_T1_BEM': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248_T1_BEM.py',
         'steps': ('source/make_bem_surfaces',),
-        'task': None,
-        'env': {}
     },
     'ds000248_FLASH_BEM': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248_FLASH_BEM.py',
         'steps': ('source/make_bem_surfaces',),
-        'task': None,
-        'env': {}
     },
     'ds000248_coreg_surfaces': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248_coreg_surfaces.py',
         'steps': ('freesurfer/coreg_surfaces',),
-        'task': None,
-        'env': {}
     },
     'ds000248_no_mri': {
-        'dataset': 'ds000248',
-        'config': 'config_ds000248_no_mri.py',
         'steps': ('preprocessing', 'sensor', 'source', 'report'),
-        'task': None,
-        'env': {}
     },
     'ds001810': {
-        'dataset': 'ds001810',
-        'config': 'config_ds001810.py',
         'steps': ('preprocessing', 'preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
-    },
-    'eeg_matchingpennies': {
-        'dataset': 'eeg_matchingpennies',
-        'config': 'config_eeg_matchingpennies.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
     },
     'ds003104': {
-        'dataset': 'ds003104',
-        'config': 'config_ds003104.py',
         'steps': ('preprocessing', 'sensor',  'source', 'report'),
-        'task': None,
-        'env': {}
-    },
-    'ds000117': {
-        'dataset': 'ds000117',
-        'config': 'config_ds000117.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
-    },
-    'ds003775': {
-        'dataset': 'ds003775',
-        'config': 'config_ds003775.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
-        'task': None,
-        'env': {}
     },
     'ERP_CORE_N400': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'N400',
-        'env': {}
     },
     'ERP_CORE_ERN': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'ERN',
-        'env': {}
     },
     'ERP_CORE_LRP': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'LRP',
-        'env': {}
     },
     'ERP_CORE_MMN': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'MMN',
-        'env': {}
     },
     'ERP_CORE_N2pc': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'N2pc',
-        'env': {}
     },
     'ERP_CORE_N170': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'N170',
-        'env': {}
     },
     'ERP_CORE_P3': {
         'dataset': 'ERP_CORE',
         'config': 'config_ERP_CORE.py',
-        'steps': ('preprocessing', 'sensor', 'report'),
         'task': 'P3',
-        'env': {}
     }
 }
 
@@ -215,24 +143,33 @@ def run_tests(test_suite, download):
     for dataset, test_options in test_suite.items():
         # export the environment variables
         os.environ['DATASET'] = dataset
-        if test_options['env']:
+        if 'env' in test_options:
             os.environ.update(test_options['env'])
 
-        config_path = (study_template_dir / 'tests' / 'configs' /
-                       test_options['config'])
+        config = test_options.get('config', f'config_{dataset}.py')
+        config_path = study_template_dir / 'tests' / 'configs' / config
 
         # Fetch the data.
+        dataset_name = test_options.get('dataset', dataset.split('_')[0])
         if download:
-            fetch(test_options['dataset'])
+            fetch(dataset_name)
 
         # XXX Workaround for buggy date in ds000247. Remove this and the
         # XXX file referenced here once fixed!!!
+        fix_path = Path(__file__).parent
         if dataset == 'ds000247':
             shutil.copy(
-                src=Path(__file__).parent / 'ds000247_scans.tsv',
+                src=fix_path / 'ds000247_scans.tsv',
                 dst=Path('~/mne_data/ds000247/sub-0002/ses-01/'
                          'sub-0002_ses-01_scans.tsv').expanduser()
             )
+        # XXX Workaround for buggy participant_id in ds001971
+        elif dataset == 'ds001971':
+            shutil.copy(
+                src=fix_path / 'ds001971_participants.tsv',
+                dst=Path('~/mne_data/ds001971/participants.tsv').expanduser()
+            )
+
 
         # Test the `--n_jobs` parameter
         if dataset == 'ds000117':
@@ -241,8 +178,9 @@ def run_tests(test_suite, download):
             n_jobs = None
 
         # Run the tests.
-        steps = test_options['steps']
-        task = test_options['task']
+        steps = test_options.get(
+            'steps', ('preprocessing', 'sensor', 'report'))
+        task = test_options.get('task', None)
 
         run_script = study_template_dir / 'run.py'
         # We need to adjust sys.argv so we can pass "command line arguments"
diff --git a/tests/test_documented.py b/tests/test_documented.py
--- a/tests/test_documented.py
+++ b/tests/test_documented.py
@@ -2,11 +2,19 @@
 import ast
 from pathlib import Path
 import os
+import sys
+import yaml
 
-root_path = Path(__file__).parent.parent
+test_dir_path = Path(__file__).parent
+root_path = test_dir_path.parent
 
+sys.path.insert(0, str(test_dir_path))
 
-def test_documented():
+from datasets import DATASET_OPTIONS
+from run_tests import TEST_SUITE
+
+
+def test_options_documented():
     """Test that all options are suitably documented."""
     # use ast to parse config.py for assignments
     with open(root_path / "config.py", "r") as fid:
@@ -41,3 +49,82 @@ def test_documented():
                     in_doc.add(val)
     assert in_doc.difference(in_config) == set(), "Extra values in doc"
     assert in_config.difference(in_doc) == set(), "Values missing from doc"
+
+
+
+
+def test_datasets_in_doc():
+    """Test that all datasets in tests are in the doc."""
+    # There are four things to keep in sync:
+    #
+    # 1. CircleCI caches, tests, etc. (actually multiple things!)
+    # 2. docs/mkdocs.yml
+    # 3. tests/datasets.py:DATASET_OPTIONS (imported above)
+    # 4. tests/run_tests.py:TEST_SUITE (imported above)
+    #
+    # So let's make sure they stay in sync.
+
+    # 1. Read cache, test, etc. entries from CircleCI
+    with open(root_path / '.circleci' / 'config.yml', 'r') as fid:
+        circle_yaml = yaml.safe_load(fid.read())
+    caches = [
+        job[6:] for job in circle_yaml['jobs'] if job.startswith('cache_')
+    ]
+    assert len(caches) == len(set(caches))
+    caches = set(caches)
+    tests = [
+        job[5:] for job in circle_yaml['jobs'] if job.startswith('test_')
+    ]
+    assert len(tests) == len(set(tests))
+    tests = set(tests)
+    # TODO: Go through circle_yaml['workflows']['commit']['jobs'] and
+    # make sure everything is consitent there. It's going to be a bit of a
+    # pain but probably worth it to be safe...
+
+    # 3. Read examples from docs (being careful about tags we can't read)
+    class SafeLoaderIgnoreUnknown(yaml.SafeLoader):
+        def ignore_unknown(self, node):
+            return None
+
+    SafeLoaderIgnoreUnknown.add_constructor(
+        None, SafeLoaderIgnoreUnknown.ignore_unknown)
+
+    with open(root_path / 'docs' / 'mkdocs.yml', 'r') as fid:
+        examples = yaml.load(fid.read(), Loader=SafeLoaderIgnoreUnknown)
+    examples = [n for n in examples['nav'] if list(n)[0] == 'Examples'][0]
+    examples = [ex for ex in examples['Examples'] if isinstance(ex, str)]
+    examples = [ex.split('/')[-1].split('.')[0] for ex in examples]
+    assert len(examples) == len(set(examples))
+    examples = set(examples)
+
+    # 4. DATASET_OPTIONS
+    dataset_names = list(DATASET_OPTIONS)
+    assert len(dataset_names) == len(set(dataset_names))
+
+    # 5. TEST_SUITE
+    test_names = list(TEST_SUITE)
+    assert len(test_names) == len(set(test_names))
+
+    # Some have been split into multiple test runs, so trim down to the same
+    # set as caches
+    for key in ('ERP_CORE', 'ds000248'):
+        tests = set(
+            job if not job.startswith(key) else job[:len(key)]
+            for job in tests
+        )
+        dataset_names = set(
+            name if not name.startswith(key) else name[:len(key)]
+            for name in dataset_names
+        )
+        examples = set(
+            ex if not ex.startswith(key) else ex[:len(key)]
+            for ex in examples
+        )
+        test_names = set(
+            test if not test.startswith(key) else test[:len(key)]
+            for test in test_names
+        )
+    assert tests == caches, 'CircleCI tests != CircleCI caches'
+    assert tests == examples, 'CircleCI tests != docs/mkdocs.yml Examples'
+    assert tests == dataset_names, 'CircleCI tests != tests/datasets.py'
+    assert tests == test_names, 'CircleCI tests != tests/run_tests.py'