Repository: mne-tools/mne-bids

Issue:
Make get_matched_empty_room() more robust
### Describe the problem
Initially reported at https://github.com/mne-tools/mne-study-template/issues/121

So I'm working on emptyroom processing, and I wanted to test my code with `ds000246` and `ds000248`. Turns out that mne-bids' `get_matched_empty_room()` cannot discover the empty-room recordings because they're stored without the `ses` entity in their path:

https://github.com/OpenNeuroDatasets/ds000246/tree/master/sub-emptyroom/meg
https://github.com/OpenNeuroDatasets/ds000248/tree/master/sub-emptyroom/meg

#### Describe your solution
If the empty-room data doesn't have a `ses` entity, walk through `sub-emptyroom/meg/`, read all data files one by one and extract `info['meas_date']`. Find the file corresponding to the date closest to the experimental measurement.

Edit:
We can use `mne.io.read_info()`.


Discussion:
I thought that BIDS datasets should always have `ses-<session_date>`. I don't how well your solution will work for non-fif files. Quoting the specification:

> This data is stored inside a subject folder named sub-emptyroom. The session label SHOULD be that of the date of the empty-room recording (e.g. ses-YYYYMMDD).

the SHOULD means it's not optional. This should be a validator issue rather than an `mne-bids` issue.
good lord, it's a SHOULD and not a MUST :( I don't even know why SHOULD exists. It might as well be called OPTIONAL
> I thought that BIDS datasets should always have `ses-<session_date>`.

As you figured: yes, they SHOULD. That's exactly the problem LOL



> I don't how well your solution will work for non-fif files.

What do you mean?
> I don't even know why SHOULD exists. It might as well be called OPTIONAL

It's easier to use in a sentence :)
> What do you mean?

I mean I don't know if the measurement date is stored in the files for all the 5 or 6 MEG manufacturers. Might be worth looking into the MNE readers to check that.
> It's easier to use in a sentence :)

I always see SHOULD as a synonym to "PLEASE FUCKING DO IT", whereas OPTIONAL is truly optional to me :smile: 
@jasmainak 
> I mean I don't know if the measurement date is stored in the files for all the 5 or 6 MEG manufacturers. Might be worth looking into the MNE readers to check that.

I would say we should check in the following order:

- `scans.tsv`
- filename parsing
- file content parsing

If none of those succeeds, there's nothing we can do, and the user will have to manually provide the emptyroom data.

Resolving patch:
diff --git a/mne_bids/read.py b/mne_bids/read.py
--- a/mne_bids/read.py
+++ b/mne_bids/read.py
@@ -10,6 +10,7 @@
 from datetime import datetime
 import glob
 import json
+import pathlib
 
 import numpy as np
 import mne
@@ -25,8 +26,8 @@
 from mne_bids.utils import (_parse_bids_filename, _extract_landmarks,
                             _find_matching_sidecar, _parse_ext,
                             _get_ch_type_mapping, make_bids_folders,
-                            make_bids_basename, _estimate_line_freq,
-                            _get_kinds_for_sub)
+                            _gen_bids_basename,
+                            _estimate_line_freq, _get_kinds_for_sub)
 
 reader = {'.con': io.read_raw_kit, '.sqd': io.read_raw_kit,
           '.fif': io.read_raw_fif, '.pdf': io.read_raw_bti,
@@ -522,53 +523,128 @@ def get_matched_empty_room(bids_basename, bids_root):
         The basename corresponding to the best-matching empty-room measurement.
         Returns None if none was found.
     """
-    kind = 'meg'
+    kind = 'meg'  # We're only concerned about MEG data here
     bids_fname = _make_bids_fname(bids_basename=bids_basename,
                                   bids_root=bids_root, kind=kind)
     _, ext = _parse_ext(bids_fname)
+    if ext == '.fif':
+        extra_params = dict(allow_maxshield=True)
+    else:
+        extra_params = None
 
     raw = read_raw_bids(bids_basename=bids_basename, bids_root=bids_root,
-                        kind='meg')
+                        kind=kind, extra_params=extra_params)
     if raw.info['meas_date'] is None:
-        raise ValueError('Measurement date not available. Cannot get matching'
-                         ' empty room file')
+        raise ValueError('The provided recording does not have a measurement '
+                         'date set. Cannot get matching empty-room file.')
 
     ref_date = raw.info['meas_date']
     if not isinstance(ref_date, datetime):
         # for MNE < v0.20
         ref_date = datetime.fromtimestamp(raw.info['meas_date'][0])
-    search_path = make_bids_folders(bids_root=bids_root, subject='emptyroom',
-                                    session='**', make_dir=False)
-    search_path = op.join(search_path, '**', '**%s' % ext)
-    er_fnames = glob.glob(search_path)
-
-    best_er_fname = None
-    min_seconds = np.inf
-    for er_fname in er_fnames:
+
+    emptyroom_dir = pathlib.Path(make_bids_folders(bids_root=bids_root,
+                                                   subject='emptyroom',
+                                                   make_dir=False))
+
+    if not emptyroom_dir.exists():
+        return None
+
+    # Find the empty-room recording sessions.
+    emptyroom_session_dirs = [x for x in emptyroom_dir.iterdir()
+                              if x.is_dir() and str(x.name).startswith('ses-')]
+    if not emptyroom_session_dirs:  # No session sub-directories found
+        emptyroom_session_dirs = [emptyroom_dir]
+
+    # Now try to discover all recordings inside the session directories.
+
+    allowed_extensions = list(reader.keys())
+    # `.pdf` is just a "virtual" extension for BTi data (which is stored inside
+    # a dedicated directory that doesn't have an extension)
+    del allowed_extensions[allowed_extensions.index('.pdf')]
+
+    candidate_er_fnames = []
+    for session_dir in emptyroom_session_dirs:
+        dir_contents = glob.glob(op.join(session_dir, kind,
+                                         f'sub-emptyroom_*_{kind}*'))
+        for item in dir_contents:
+            item = pathlib.Path(item)
+            if ((item.suffix in allowed_extensions) or
+                    (not item.suffix and item.is_dir())):  # Hopefully BTi?
+                candidate_er_fnames.append(item.name)
+
+    # Walk through recordings, trying to extract the recording date:
+    # First, from the filename; and if that fails, from `info['meas_date']`.
+    best_er_basename = None
+    min_delta_t = np.inf
+    date_tie = False
+
+    failed_to_get_er_date_count = 0
+    for er_fname in candidate_er_fnames:
         params = _parse_bids_filename(er_fname, verbose=False)
-        dt = datetime.strptime(params['ses'], '%Y%m%d')
-        dt = dt.replace(tzinfo=ref_date.tzinfo)
-        delta_t = dt - ref_date
-        if abs(delta_t.total_seconds()) < min_seconds:
-            min_seconds = abs(delta_t.total_seconds())
-            best_er_fname = er_fname
-
-    if best_er_fname is None:
-        er_basename = None
-    else:
-        params = _parse_bids_filename(best_er_fname, verbose='warning')
-        er_basename = make_bids_basename(
-            subject=params.get('sub', None),
+        er_meas_date = None
+
+        er_basename = _gen_bids_basename(
+            subject='emptyroom',
             session=params.get('ses', None),
             task=params.get('task', None),
             acquisition=params.get('acq', None),
             run=params.get('run', None),
             processing=params.get('proc', None),
             recording=params.get('recording', None),
-            space=params.get('space', None)
+            space=params.get('space', None),
+            # BIDS specification does not enforce use of ses-YYYYMMDD and
+            # task-emptyroom entities.
+            on_invalid_er_session='continue',
+            on_invalid_er_task='continue'
         )
 
-    return er_basename
+        if 'ses' in params:  # Try to extract date from filename.
+            try:
+                er_meas_date = datetime.strptime(params['ses'], '%Y%m%d')
+            except (ValueError, TypeError):
+                # There is a session in the filename, but it doesn't encode a
+                # valid date.
+                pass
+
+        if er_meas_date is None:  # No luck so far! Check info['meas_date']
+            _, ext = _parse_ext(er_fname)
+            if ext == '.fif':
+                extra_params = dict(allow_maxshield=True)
+            else:
+                extra_params = None
+
+            er_raw = read_raw_bids(bids_basename=er_basename,
+                                   bids_root=bids_root,
+                                   kind=kind,
+                                   extra_params=extra_params)
+
+            er_meas_date = er_raw.info['meas_date']
+            if er_meas_date is None:  # There's nothing we can do.
+                failed_to_get_er_date_count += 1
+                continue
+
+        er_meas_date = er_meas_date.replace(tzinfo=ref_date.tzinfo)
+        delta_t = er_meas_date - ref_date
+
+        if abs(delta_t.total_seconds()) == min_delta_t:
+            date_tie = True
+        elif abs(delta_t.total_seconds()) < min_delta_t:
+            min_delta_t = abs(delta_t.total_seconds())
+            best_er_basename = er_basename
+            date_tie = False
+
+    if failed_to_get_er_date_count > 0:
+        msg = (f'Could not retrieve the empty-room measurement date from '
+               f'a total of {failed_to_get_er_date_count} recording(s).')
+        warn(msg)
+
+    if date_tie:
+        msg = ('Found more than one matching empty-room measurement with the '
+               'same recording date. Selecting the first match.')
+        warn(msg)
+
+    return best_er_basename
 
 
 def get_head_mri_trans(bids_basename, bids_root):
diff --git a/mne_bids/utils.py b/mne_bids/utils.py
--- a/mne_bids/utils.py
+++ b/mne_bids/utils.py
@@ -804,8 +804,8 @@ def _gen_bids_basename(*, subject=None, session=None, task=None,
                        recording=None, space=None, prefix=None, suffix=None,
                        on_invalid_er_session='raise',
                        on_invalid_er_task='raise'):
-    if on_invalid_er_session not in ['raise', 'warn']:
-        msg = (f'on_invalid_er_session must be raise or warn, '
+    if on_invalid_er_session not in ['raise', 'warn', 'continue']:
+        msg = (f'on_invalid_er_session must be raise, warn, or continue, '
                f'but received: {on_invalid_er_session}')
         raise ValueError(msg)
 
@@ -834,14 +834,15 @@ def _gen_bids_basename(*, subject=None, session=None, task=None,
         raise ValueError("At least one parameter must be given.")
 
     if subject == 'emptyroom':
-        if task != 'noise' and on_invalid_er_task != 'continue':
+        if task != 'noise':
             msg = (f'task must be "noise" if subject is "emptyroom", but '
                    f'received: {task}')
             if on_invalid_er_task == 'raise':
                 raise ValueError(msg)
-            else:
+            elif on_invalid_er_task == 'warn':
                 logger.critical(msg)
-
+            else:
+                pass
         try:
             datetime.strptime(session, '%Y%m%d')
         except (ValueError, TypeError):
@@ -849,10 +850,12 @@ def _gen_bids_basename(*, subject=None, session=None, task=None,
                    f'YYYYMMDD, but received: {session}')
             if on_invalid_er_session == 'raise':
                 raise ValueError(msg)
-            else:
+            elif on_invalid_er_session == 'warn':
                 msg = (f'{msg}. Will proceed anyway, but you should consider '
                        f'fixing your dataset.')
                 logger.critical(msg)
+            else:
+                pass
 
     basename = []
     for key, val in order.items():
diff --git a/mne_bids/write.py b/mne_bids/write.py
--- a/mne_bids/write.py
+++ b/mne_bids/write.py
@@ -987,12 +987,12 @@ def write_raw_bids(raw, bids_basename, bids_root, events_data=None,
         subject=subject_id, session=session_id, acquisition=acquisition,
         suffix='electrodes.tsv', prefix=data_path,
         on_invalid_er_task=on_invalid_er_task)
+
     # For the remaining files, we can use make_bids_basename() as usual.
     participants_tsv_fname = make_bids_basename(prefix=bids_root,
                                                 suffix='participants.tsv')
     participants_json_fname = make_bids_basename(prefix=bids_root,
                                                  suffix='participants.json')
-
     sidecar_fname = make_bids_basename(
         subject=subject_id, session=session_id, task=task, run=run,
         acquisition=acquisition, suffix='%s.json' % kind, prefix=data_path)
@@ -1003,6 +1003,7 @@ def write_raw_bids(raw, bids_basename, bids_root, events_data=None,
     channels_fname = make_bids_basename(
         subject=subject_id, session=session_id, task=task, run=run,
         acquisition=acquisition, suffix='channels.tsv', prefix=data_path)
+
     if ext not in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.con',
                    '.sqd']:
         bids_raw_folder = bids_fname.split('.')[0]


Test patch:
diff --git a/mne_bids/tests/test_read.py b/mne_bids/tests/test_read.py
--- a/mne_bids/tests/test_read.py
+++ b/mne_bids/tests/test_read.py
@@ -7,6 +7,7 @@
 import os.path as op
 from datetime import datetime, timezone
 from pathlib import Path
+from distutils.version import LooseVersion
 
 import pytest
 import shutil as sh
@@ -27,14 +28,15 @@
 from mne.utils import assert_dig_allclose
 from mne.datasets import testing, somato
 
-from mne_bids import get_matched_empty_room, make_bids_basename
+from mne_bids import (get_matched_empty_room, make_bids_basename,
+                      make_bids_folders)
 from mne_bids.config import MNE_STR_TO_FRAME
 from mne_bids.read import (read_raw_bids,
                            _read_raw, get_head_mri_trans,
                            _handle_events_reading, _handle_info_reading)
 from mne_bids.tsv_handler import _to_tsv, _from_tsv
 from mne_bids.utils import (_find_matching_sidecar, _update_sidecar,
-                            _write_json)
+                            _write_json, _gen_bids_basename)
 from mne_bids.write import write_anat, write_raw_bids
 
 subject_id = '01'
@@ -647,7 +649,80 @@ def test_get_matched_empty_room():
         raw.annotations.orig_time = None
     anonymize_info(raw.info)
     write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
-    with pytest.raises(ValueError, match='Measurement date not available'):
+    with pytest.raises(ValueError, match='The provided recording does not '
+                                         'have a measurement date set'):
+        get_matched_empty_room(bids_basename=bids_basename,
+                               bids_root=bids_root)
+
+
+@pytest.mark.filterwarnings(warning_str['channel_unit_changed'])
+def test_get_matched_emptyroom_ties():
+    """Test that we receive a warning on a date tie."""
+    bids_root = _TempDir()
+    session = '20010101'
+    er_dir = make_bids_folders(subject='emptyroom', session=session,
+                               kind='meg', bids_root=bids_root)
+
+    meas_date = (datetime
+                 .strptime(session, '%Y%m%d')
+                 .replace(tzinfo=timezone.utc))
+
+    raw = mne.io.read_raw_fif(raw_fname)
+    er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif')
+    raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True)
+    er_raw = mne.io.read_raw_fif(er_raw_fname)
+
+    if check_version('mne', '0.20'):
+        raw.set_meas_date(meas_date)
+        er_raw.set_meas_date(meas_date)
+    else:
+        raw.info['meas_date'] = (meas_date.timestamp(), 0)
+        er_raw.info['meas_date'] = (meas_date.timestamp(), 0)
+
+    write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
+    er_basename_1 = _gen_bids_basename(subject='emptyroom', session=session,
+                                       task=None,
+                                       on_invalid_er_task='continue')
+    er_basename_2 = make_bids_basename(subject='emptyroom', session=session,
+                                       task='noise')
+    er_raw.save(op.join(er_dir, f'{er_basename_1}_meg.fif'))
+    er_raw.save(op.join(er_dir, f'{er_basename_2}_meg.fif'))
+
+    with pytest.warns(RuntimeWarning, match='Found more than one'):
+        get_matched_empty_room(bids_basename=bids_basename,
+                               bids_root=bids_root)
+
+
+@pytest.mark.skipif(LooseVersion(mne.__version__) < LooseVersion('0.21'),
+                    reason="requires mne 0.21.dev0 or higher")
+@pytest.mark.filterwarnings(warning_str['channel_unit_changed'])
+def test_get_matched_emptyroom_no_meas_date():
+    """Test that we warn if measurement date can be read or inferred."""
+    bids_root = _TempDir()
+    er_session = 'mysession'
+    er_meas_date = None
+
+    er_dir = make_bids_folders(subject='emptyroom', session=er_session,
+                               kind='meg', bids_root=bids_root)
+    er_basename = _gen_bids_basename(subject='emptyroom',
+                                     session=er_session,
+                                     task='noise',
+                                     on_invalid_er_session='continue')
+    raw = mne.io.read_raw_fif(raw_fname)
+    er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif')
+    raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True)
+    er_raw = mne.io.read_raw_fif(er_raw_fname)
+    er_raw.set_meas_date(er_meas_date)
+    er_raw.save(op.join(er_dir, f'{er_basename}_meg.fif'), overwrite=True)
+
+    # Write raw file data using mne-bids, and remove participants.tsv
+    # as it's incomplete (doesn't contain the emptyroom subject we wrote
+    # manually using MNE's Raw.save() above)
+    raw = mne.io.read_raw_fif(raw_fname)
+    write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
+    os.remove(op.join(bids_root, 'participants.tsv'))
+
+    with pytest.warns(RuntimeWarning, match='Could not retrieve .* date'):
         get_matched_empty_room(bids_basename=bids_basename,
                                bids_root=bids_root)