Repository: mne-tools/mne-bids

Issue:
Read with underscore
Thanks for contributing. If this is your first time,
make sure to read [contributing.md](https://github.com/mne-tools/mne-bids/blob/master/CONTRIBUTING.md)

PR Description
--------------

Don't include BIDS directory name in parsing because it can cause issues.

Merge checklist
---------------

Maintainer, please confirm the following before merging:

- [ ] All comments resolved
- [ ] This is not your own PR
- [ ] All CIs are happy
- [ ] PR title starts with [MRG]
- [ ] [whats_new.rst](https://github.com/mne-tools/mne-bids/blob/master/doc/whats_new.rst) is updated
- [ ] PR description includes phrase "closes <#issue-number>"
- [ ] Commit history does not contain any merge commits

Read with underscore
Thanks for contributing. If this is your first time,
make sure to read [contributing.md](https://github.com/mne-tools/mne-bids/blob/master/CONTRIBUTING.md)

PR Description
--------------

Don't include BIDS directory name in parsing because it can cause issues.

Merge checklist
---------------

Maintainer, please confirm the following before merging:

- [ ] All comments resolved
- [ ] This is not your own PR
- [ ] All CIs are happy
- [ ] PR title starts with [MRG]
- [ ] [whats_new.rst](https://github.com/mne-tools/mne-bids/blob/master/doc/whats_new.rst) is updated
- [ ] PR description includes phrase "closes <#issue-number>"
- [ ] Commit history does not contain any merge commits



Discussion:
Just a note: It'd be great if you could edit your original post to include:

`closes #230`, because then upon merge of this PR, your issue will be automatically closed.

Also, you can delete the following:
> Thanks for contributing. If this is your first time, make sure to read contributing.md
Just a note: It'd be great if you could edit your original post to include:

`closes #230`, because then upon merge of this PR, your issue will be automatically closed.

Also, you can delete the following:
> Thanks for contributing. If this is your first time, make sure to read contributing.md

Resolving patch:
diff --git a/mne_bids/read.py b/mne_bids/read.py
--- a/mne_bids/read.py
+++ b/mne_bids/read.py
@@ -13,7 +13,7 @@
 import numpy as np
 import mne
 from mne import io
-from mne.utils import has_nibabel
+from mne.utils import has_nibabel, logger
 from mne.coreg import fit_matched_points
 from mne.transforms import apply_trans
 
@@ -76,6 +76,7 @@ def _handle_events_reading(events_fname, raw):
 
     Handle onset, duration, and description of each event.
     """
+    logger.info('Reading events from {}.'.format(events_fname))
     events_dict = _from_tsv(events_fname)
 
     # Get the descriptions of the events
@@ -112,6 +113,7 @@ def _handle_channels_reading(channels_fname, bids_fname, raw):
 
     Updates status (bad) and types of channels.
     """
+    logger.info('Reading channel info from {}.'.format(channels_fname))
     channels_dict = _from_tsv(channels_fname)
 
     # First, make sure that ordering of names in channels.tsv matches the
@@ -195,6 +197,7 @@ def read_raw_bids(bids_fname, bids_root, verbose=True):
     """
     # Full path to data file is needed so that mne-bids knows
     # what is the modality -- meg, eeg, ieeg to read
+    bids_fname = op.basename(bids_fname)
     bids_basename = '_'.join(bids_fname.split('_')[:-1])
     kind = bids_fname.split('_')[-1].split('.')[0]
     _, ext = _parse_ext(bids_fname)
diff --git a/mne_bids/utils.py b/mne_bids/utils.py
--- a/mne_bids/utils.py
+++ b/mne_bids/utils.py
@@ -13,6 +13,7 @@
 import warnings
 import json
 import shutil as sh
+import re
 from datetime import datetime
 from collections import defaultdict
 
@@ -132,19 +133,22 @@ def _parse_ext(raw_fname, verbose=False):
     return fname, ext
 
 
+# This regex matches key-val pairs. Any characters are allowed in the key and
+# the value, except these special symbols: - _ . \ /
+param_regex = re.compile(r'([^-_\.\\\/]+)-([^-_\.\\\/]+)')
+
+
 def _parse_bids_filename(fname, verbose):
     """Get dict from BIDS fname."""
     keys = ['sub', 'ses', 'task', 'acq', 'run', 'proc', 'run', 'space',
             'recording', 'kind']
     params = {key: None for key in keys}
-    entities = fname.split('_')
     idx_key = 0
-    for entity in entities:
-        assert '-' in entity
-        key, value = entity.split('-')
+    for match in re.finditer(param_regex, op.basename(fname)):
+        key, value = match.groups()
         if key not in keys:
             raise KeyError('Unexpected entity ''%s'' found in filename ''%s'''
-                           % (entity, fname))
+                           % (key, fname))
         if keys.index(key) < idx_key:
             raise ValueError('Entities in filename not ordered correctly.'
                              ' "%s" should have occured earlier in the '
@@ -349,6 +353,55 @@ def _extract_landmarks(dig):
     return coords
 
 
+def _find_best_candidates(params, candidate_list):
+    """Return the best candidate, based on the number of param matches.
+
+    Assign each candidate a score, based on how many entities are shared with
+    the ones supplied in the `params` parameter. The candidate with the highest
+    score wins. Candidates with entities that conflict with the supplied
+    `params` are disqualified.
+
+    Parameters
+    ----------
+    params : dict
+        The entities that the candidate should match.
+    candidate_list : list of str
+        A list of candidate filenames.
+
+    Returns
+    -------
+    best_candidates : list of str
+        A list of all the candidate filenames that are tied for first place.
+        Hopefully, the list will have a length of one.
+    """
+    params = {key: value for key, value in params.items() if value is not None}
+
+    best_candidates = []
+    best_n_matches = 0
+    for candidate in candidate_list:
+        n_matches = 0
+        candidate_disqualified = False
+        candidate_params = _parse_bids_filename(candidate, verbose=False)
+        for entity, value in params.items():
+            if entity in candidate_params:
+                if candidate_params[entity] is None:
+                    continue
+                elif candidate_params[entity] == value:
+                    n_matches += 1
+                else:
+                    # Incompatible entity found, candidate is disqualified
+                    candidate_disqualified = True
+                    break
+        if not candidate_disqualified:
+            if n_matches > best_n_matches:
+                best_n_matches = n_matches
+                best_candidates = [candidate]
+            elif n_matches == best_n_matches:
+                best_candidates.append(candidate)
+
+    return best_candidates
+
+
 def _find_matching_sidecar(bids_fname, bids_root, suffix, allow_fail=False):
     """Try to find a sidecar file with a given suffix for a data file.
 
@@ -371,31 +424,38 @@ def _find_matching_sidecar(bids_fname, bids_root, suffix, allow_fail=False):
         and no sidecar_fname was found
 
     """
-    bids_fname = op.basename(bids_fname)
+    params = _parse_bids_filename(bids_fname, verbose=False)
 
     # We only use subject and session as identifier, because all other
     # parameters are potentially not binding for metadata sidecar files
-    if 'ses' in bids_fname:
-        search_str = '_'.join(bids_fname.split('_')[:2])
-    else:
-        # only sub, no ses
-        search_str = '_'.join(bids_fname.split('_')[:1])
+    search_str = 'sub-' + params['sub']
+    if params['ses'] is not None:
+        search_str += '_ses-' + params['ses']
 
-    # find the sidecar file, doing a recursive glob from the bids_root
+    # Find all potential sidecar files, doing a recursive glob from bids_root
     search_str = op.join(bids_root, '**', search_str + '*' + suffix)
     candidate_list = glob.glob(search_str, recursive=True)
-    if len(candidate_list) != 1:
+    best_candidates = _find_best_candidates(params, candidate_list)
+
+    if len(best_candidates) == 1:
+        # Success
+        return best_candidates[0]
+
+    # We failed. Construct a helpful error message.
+    # If this was expected, simply return None, otherwise, raise an exception.
+    msg = None
+    if len(best_candidates) == 0:
+        msg = ('Did not find any {} file associated with {}.'
+               .format(suffix, bids_fname))
+    elif len(best_candidates) > 1:
+        # More than one candidates were tied for best match
         msg = ('Expected to find a single {} file associated with '
-               '{}, but found {}: "{}".\n\nThe search_str was "{}"'
+               '{}, but found {}: "{}".'
                .format(suffix, bids_fname, len(candidate_list),
-                       candidate_list, search_str))
-        # We failed. If this was expected, simply return None
-        if allow_fail:
-            warnings.warn(msg)
-            return None
-        else:
-            raise RuntimeError(msg)
-
+                       candidate_list))
+    msg += '\n\nThe search_str was "{}"'.format(search_str)
+    if allow_fail:
+        warnings.warn(msg)
+        return None
     else:
-        sidecar_fname = candidate_list[0]
-    return sidecar_fname
+        raise RuntimeError(msg)


Test patch:
diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py
--- a/mne_bids/tests/test_utils.py
+++ b/mne_bids/tests/test_utils.py
@@ -17,7 +17,8 @@
 from mne_bids.utils import (_check_types, print_dir_tree, _age_on_date,
                             _infer_eeg_placement_scheme, _handle_kind,
                             _find_matching_sidecar, _parse_ext,
-                            _get_ch_type_mapping)
+                            _get_ch_type_mapping, _parse_bids_filename,
+                            _find_best_candidates)
 
 
 base_path = op.join(op.dirname(mne.__file__), 'io')
@@ -137,6 +138,23 @@ def test_parse_ext():
     assert ext == '.nii.gz'
 
 
+@pytest.mark.parametrize('fname', [
+    'sub-01_ses-02_task-test_run-3_meg.fif',
+    'sub-01_ses-02_task-test_run-3.fif',
+    'sub-01_ses-02_task-test_run-3',
+    '/bids_root/sub-01/ses-02/meg/sub-01_ses-02_task-test_run-3_meg.fif',
+])
+def test_parse_bids_filename(fname):
+    """Test parsing entities from a bids filename."""
+    params = _parse_bids_filename(fname, verbose=False)
+    assert params['sub'] == '01'
+    assert params['ses'] == '02'
+    assert params['run'] == '3'
+    assert params['task'] == 'test'
+    assert list(params.keys()) == ['sub', 'ses', 'task', 'acq', 'run', 'proc',
+                                   'space', 'recording', 'kind']
+
+
 def test_age_on_date():
     """Test whether the age is determined correctly."""
     bday = datetime(1994, 1, 26)
@@ -175,6 +193,28 @@ def test_infer_eeg_placement_scheme():
     assert placement_scheme == 'n/a'
 
 
+@pytest.mark.parametrize('candidate_list, best_candidates', [
+    # Only one candidate
+    (['sub-01_ses-02'], ['sub-01_ses-02']),
+
+    # Two candidates, but the second matches on more entities
+    (['sub-01', 'sub-01_ses-02'], ['sub-01_ses-02']),
+
+    # No candidates match
+    (['sub-02_ses-02', 'sub-01_ses-01'], []),
+
+    # First candidate is disqualified (session doesn't match)
+    (['sub-01_ses-01', 'sub-01_ses-02'], ['sub-01_ses-02']),
+
+    # Multiple equally good candidates
+    (['sub-01_run-01', 'sub-01_run-02'], ['sub-01_run-01', 'sub-01_run-02']),
+])
+def test_find_best_candidates(candidate_list, best_candidates):
+    """Test matching of candidate sidecar files."""
+    params = dict(sub='01', ses='02', acq=None)
+    assert _find_best_candidates(params, candidate_list) == best_candidates
+
+
 def test_find_matching_sidecar():
     """Test finding a sidecar file from a BIDS dir."""
     # First write a BIDS dir
@@ -199,15 +239,14 @@ def test_find_matching_sidecar():
                                            'coordsystem.json')
     expected_file = op.join('sub-01', 'ses-01', 'meg',
                             'sub-01_ses-01_acq-01_coordsystem.json')
-
     assert sidecar_fname.endswith(expected_file)
 
-    # Find multiple sidecars, triggering an error
+    # Find multiple sidecars, tied in score, triggering an error
     with pytest.raises(RuntimeError, match='Expected to find a single'):
         open(sidecar_fname.replace('coordsystem.json',
                                    '2coordsystem.json'), 'w').close()
         _find_matching_sidecar(bids_basename, output_path, 'coordsystem.json')
 
     # Find nothing but receive None, because we set `allow_fail` to True
-    with pytest.warns(UserWarning, match='Expected to find a single'):
+    with pytest.warns(UserWarning, match='Did not find any'):
         _find_matching_sidecar(bids_basename, output_path, 'foo.bogus', True)
diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py
--- a/mne_bids/tests/test_write.py
+++ b/mne_bids/tests/test_write.py
@@ -303,7 +303,7 @@ def test_ctf():
 
     cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
-    with pytest.warns(UserWarning, match='Expected to find a single events'):
+    with pytest.warns(UserWarning, match='Did not find any events'):
         raw = read_raw_bids(bids_basename + '_meg.ds', output_path)
 
     # test to check that running again with overwrite == False raises an error
@@ -616,7 +616,7 @@ def test_write_anat():
     sh.rmtree(anat_dir)
     write_anat(output_path, subject_id, t1w_mgh, session_id)
     # Assert that we truly cannot find a sidecar
-    with pytest.raises(RuntimeError, match='Expected to find a single'):
+    with pytest.raises(RuntimeError, match='Did not find any'):
         _find_matching_sidecar('sub-01_ses-01_acq-01_T1w.nii.gz',
                                output_path, 'T1w.json')