Repository: mne-tools/mne-bids

Issue:
automatic bad channel handling
When we read in BIDS data, the `raw.info['bads']` should be automatically populated by reading it in from channels.tsv.
read_raw_bids hardcodes the dtypes to be extracted from events.tsv files
In BIDS, there is no prescription about the column order in events.tsv apart from the first two columns, which must be onset and duration. Furthermore, there can be an arbitrary amount of columns in an `events.tsv` file.

Thus hardcoding `dtypes` is a bug and needs to be fixed:

https://github.com/mne-tools/mne-bids/blob/0fa987edeef36fed8dfc04f04c50224900158f8f/mne_bids/read.py#L143-L144




Discussion:
@sappelhoff probably an easy issue for you to get back to gsoc next week :)


Resolving patch:
diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py
--- a/examples/convert_mne_sample.py
+++ b/examples/convert_mne_sample.py
@@ -10,7 +10,8 @@
 # Authors: Mainak Jas <mainak.jas@telecom-paristech.fr>
 #          Alexandre Gramfort <alexandre.gramfort@telecom-paristech.fr>
 #          Teon Brooks <teon.brooks@gmail.com>
-
+#          Stefan Appelhoff <stefan.appelhoff@mailbox.org>
+#
 # License: BSD (3-clause)
 
 ###############################################################################
@@ -65,9 +66,10 @@
 
 ###############################################################################
 # Finally, we can read the BIDS data we created as well.
-raw, events, event_id = read_raw_bids(bids_basename + '_meg.fif', output_path)
+raw = read_raw_bids(bids_basename + '_meg.fif', output_path)
 
 ###############################################################################
 # The data is already in a convenient form to create epochs and evokeds.
+events, event_id = mne.events_from_annotations(raw)
 epochs = mne.Epochs(raw, events, event_id)
 epochs['Auditory'].average().plot()
diff --git a/mne_bids/read.py b/mne_bids/read.py
--- a/mne_bids/read.py
+++ b/mne_bids/read.py
@@ -11,10 +11,12 @@
 import glob
 
 import numpy as np
+import mne
 from mne import io
 
 from .tsv_handler import _from_tsv, _drop
 from .config import ALLOWED_EXTENSIONS
+from .utils import _parse_bids_filename
 
 reader = {'.con': io.read_raw_kit, '.sqd': io.read_raw_kit,
           '.fif': io.read_raw_fif, '.pdf': io.read_raw_bti,
@@ -77,18 +79,18 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None,
     return raw
 
 
-def read_raw_bids(bids_fname, bids_root, return_events=True,
-                  verbose=True):
+def read_raw_bids(bids_fname, bids_root, verbose=True):
     """Read BIDS compatible data.
 
+    Will attempt to read associated events.tsv and channels.tsv files to
+    populate the returned raw object with raw.annotations and raw.info['bads'].
+
     Parameters
     ----------
     bids_fname : str
         Full name of the data file
     bids_root : str
         Path to root of the BIDS folder
-    return_events: bool
-        Whether to return events or not. Default is True.
     verbose : bool
         The verbosity level
 
@@ -96,18 +98,8 @@ def read_raw_bids(bids_fname, bids_root, return_events=True,
     -------
     raw : instance of Raw
         The data as MNE-Python Raw object.
-    events : ndarray, shape = (n_events, 3)
-        The first column contains the event time in samples and the third
-        column contains the event id. The second column is ignored for now but
-        typically contains the value of the trigger channel either immediately
-        before the event or immediately after.
-    event_id : dict
-        Dictionary of events in the raw data mapping from the event value
-        to an index.
 
     """
-    from .utils import _parse_bids_filename
-
     # Full path to data file is needed so that mne-bids knows
     # what is the modality -- meg, eeg, ieeg to read
     bids_basename = '_'.join(bids_fname.split('_')[:-1])
@@ -127,34 +119,50 @@ def read_raw_bids(bids_fname, bids_root, return_events=True,
         bids_raw_folder = op.join(kind_dir, bids_basename + '_%s' % kind)
         bids_fname = glob.glob(op.join(bids_raw_folder, 'c,rf*'))[0]
         config = op.join(bids_raw_folder, 'config')
+
     raw = _read_raw(bids_fname, electrode=None, hsp=None, hpi=None,
                     config=config, montage=None, verbose=None)
 
+    # Try to find an associated events.tsv to get information about the
+    # events in the recorded data
     events_fname = op.join(kind_dir, bids_basename + '_events.tsv')
+    if op.exists(events_fname):
+        events_dict = _from_tsv(events_fname)
+    else:
+        events_dict = dict()
+
+    if 'trial_type' in events_dict:
+        # Drop events unrelated to a trial type
+        events_dict = _drop(events_dict, 'n/a', 'trial_type')
+
+        # Add Events to raw as annotations
+        onsets = np.asarray(events_dict['onset'], dtype=float)
+        durations = np.asarray(events_dict['duration'], dtype=float)
+        descriptions = np.asarray(events_dict['trial_type'], dtype=str)
+        annot_from_events = mne.Annotations(onset=onsets,
+                                            duration=durations,
+                                            description=descriptions,
+                                            orig_time=raw.info['meas_date'])
+        raw.set_annotations(annot_from_events)
+
+    # Try to find an associated channels.tsv to get information about the
+    # status of present channels
+    channels_fname = op.join(kind_dir, bids_basename + '_channels.tsv')
+    if op.exists(channels_fname):
+        channels_dict = _from_tsv(channels_fname)
+    else:
+        channels_dict = dict()
 
-    if not return_events:
-        return raw
-
-    if not op.exists(events_fname):
-        raise ValueError('return_events=True but _events.tsv file'
-                         ' not found')
-
-    events, event_id = None, dict()
-    dtypes = [float, float, str, int, int]
-    events_dict = _from_tsv(events_fname, dtypes=dtypes)
-    events_dict = _drop(events_dict, 'n/a', 'trial_type')
-
-    if 'trial_type' not in events_dict:
-        raise ValueError('trial_type column is missing. Cannot'
-                         ' return events')
-
-    for idx, ev in enumerate(np.unique(events_dict['trial_type'])):
-        event_id[ev] = idx
+    # If we have a channels.tsv file, make sure there is the optional "status"
+    # column from which to infer good and bad channels
+    if 'status' in channels_dict:
+        # find bads from channels.tsv
+        bad_bool = [True if chn == 'bad' else False
+                    for chn in channels_dict['status']]
+        bads = np.asarray(channels_dict['name'])[bad_bool]
 
-    events = np.zeros((len(events_dict['onset']), 3), dtype=int)
-    events[:, 0] = np.array(events_dict['onset']) * raw.info['sfreq'] + \
-        raw.first_samp
-    events[:, 2] = np.array([event_id[ev] for ev
-                             in events_dict['trial_type']])
+        # merge with bads already present in raw data file (if there are any)
+        unique_bads = set(raw.info['bads']).union(set(bads))
+        raw.info['bads'] = list(unique_bads)
 
-    return raw, events, event_id
+    return raw


Test patch:
diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py
--- a/mne_bids/tests/test_write.py
+++ b/mne_bids/tests/test_write.py
@@ -15,6 +15,7 @@
 import pytest
 from glob import glob
 from datetime import datetime
+import platform
 
 import numpy as np
 from numpy.testing import assert_array_equal
@@ -41,13 +42,21 @@
     subject=subject_id, session=session_id, run=run, acquisition=acq,
     task=task)
 
-# for windows, shell = True is needed
-# to call npm, bids-validator etc.
-#     see: https://stackoverflow.com/questions/
-#          28891053/run-npm-commands-using-python-subprocess
+# WINDOWS issues:
+# the bids-validator development version does not work properly on Windows as
+# of 2019-06-25 --> https://github.com/bids-standard/bids-validator/issues/790
+# As a workaround, we try to get the path to the executable from an environment
+# variable VALIDATOR_EXECUTABLE ... if this is not possible we assume to be
+# using the stable bids-validator and make a direct call of bids-validator
+# also: for windows, shell = True is needed to call npm, bids-validator etc.
+# see: https://stackoverflow.com/q/28891053/5201771
 shell = False
-if os.name == 'nt':
+bids_validator_exe = ['bids-validator']
+if platform.system() == 'Windows':
     shell = True
+    exe = os.getenv('VALIDATOR_EXECUTABLE', 'n/a')
+    if 'VALIDATOR_EXECUTABLE' != 'n/a':
+        bids_validator_exe = ['node', exe]
 
 
 def test_fif():
@@ -73,7 +82,7 @@ def test_fif():
     output_path = _TempDir()
     write_raw_bids(raw, bids_basename, output_path, overwrite=False)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
     # write the same data but pretend it is empty room data:
@@ -132,7 +141,8 @@ def test_fif():
     # now force the overwrite
     write_raw_bids(raw, bids_basename2, output_path, events_data=events_fname,
                    event_id=event_id, overwrite=True)
-    raw, events, _ = read_raw_bids(bids_basename2 + '_meg.fif', output_path)
+    raw = read_raw_bids(bids_basename2 + '_meg.fif', output_path)
+    events, _ = mne.events_from_annotations(raw)
     events2 = mne.read_events(events_fname)
     events2 = events2[events2[:, 2] != 0]
     assert_array_equal(events2[:, 0], events[:, 0])
@@ -152,7 +162,7 @@ def test_fif():
     with pytest.raises(ValueError, match='raw.filenames is missing'):
         write_raw_bids(raw, bids_basename2, output_path)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
     assert op.exists(op.join(output_path, 'participants.tsv'))
@@ -202,12 +212,11 @@ def test_kit():
     write_raw_bids(raw, kit_bids_basename, output_path,
                    events_data=events_fname,
                    event_id=event_id, overwrite=False)
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
     assert op.exists(op.join(output_path, 'participants.tsv'))
 
-    raw2, events, _ = read_raw_bids(kit_bids_basename + '_meg.sqd',
-                                    output_path)
+    read_raw_bids(kit_bids_basename + '_meg.sqd', output_path)
 
     # ensure the channels file has no STI 014 channel:
     channels_tsv = make_bids_basename(
@@ -228,14 +237,15 @@ def test_kit():
     event_data = np.loadtxt(events_fname)
     # make the data the wrong number of dimensions
     event_data_3d = np.atleast_3d(event_data)
+    other_output_path = _TempDir()
     with pytest.raises(ValueError, match='two dimensions'):
-        write_raw_bids(raw, bids_basename, output_path,
+        write_raw_bids(raw, bids_basename, other_output_path,
                        events_data=event_data_3d, event_id=event_id,
                        overwrite=True)
     # remove 3rd column
     event_data = event_data[:, :2]
     with pytest.raises(ValueError, match='second dimension'):
-        write_raw_bids(raw, bids_basename, output_path,
+        write_raw_bids(raw, bids_basename, other_output_path,
                        events_data=event_data, event_id=event_id,
                        overwrite=True)
     # test correct naming of marker files
@@ -246,7 +256,7 @@ def test_kit():
                    kit_bids_basename.replace('sub-01', 'sub-%s' % subject_id2),
                    output_path, events_data=events_fname, event_id=event_id,
                    overwrite=False)
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
     # ensure the marker files are renamed correctly
     marker_fname = make_bids_basename(
@@ -282,14 +292,10 @@ def test_ctf():
     raw = mne.io.read_raw_ctf(raw_fname)
     write_raw_bids(raw, bids_basename, output_path=output_path)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
-    with pytest.raises(ValueError, match="not found"):
-        read_raw_bids(bids_basename + '_meg.ds', output_path)
-
-    raw = read_raw_bids(bids_basename + '_meg.ds', output_path,
-                        return_events=False)
+    raw = read_raw_bids(bids_basename + '_meg.ds', output_path)
 
     # test to check that running again with overwrite == False raises an error
     with pytest.raises(FileExistsError, match="already exists"):  # noqa: F821
@@ -313,11 +319,10 @@ def test_bti():
 
     assert op.exists(op.join(output_path, 'participants.tsv'))
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
-    raw = read_raw_bids(bids_basename + '_meg',
-                        output_path, return_events=False)
+    raw = read_raw_bids(bids_basename + '_meg', output_path)
 
 
 def test_vhdr():
@@ -327,21 +332,34 @@ def test_vhdr():
     raw_fname = op.join(data_path, 'test.vhdr')
 
     raw = mne.io.read_raw_brainvision(raw_fname)
+
+    # inject a bad channel
+    assert not raw.info['bads']
+    injected_bad = ['FP1']
+    raw.info['bads'] = injected_bad
+
+    # write with injected bad channels
     write_raw_bids(raw, bids_basename, output_path, overwrite=False)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
-    read_raw_bids(bids_basename + '_eeg.vhdr',
-                  output_path, return_events=False)
+    # read and also get the bad channels
+    raw = read_raw_bids(bids_basename + '_eeg.vhdr', output_path)
+
+    # Check that injected bad channel shows up in raw after reading
+    np.testing.assert_array_equal(np.asarray(raw.info['bads']),
+                                  np.asarray(injected_bad))
 
-    # Test that correct channel units are written
+    # Test that correct channel units are written ... and that bad channel
+    # is in channels.tsv
     channels_tsv_name = op.join(output_path, 'sub-' + subject_id,
                                 'ses-' + session_id, 'eeg',
                                 bids_basename + '_channels.tsv')
     data = _from_tsv(channels_tsv_name)
     assert data['units'][data['name'].index('FP1')] == 'µV'
     assert data['units'][data['name'].index('CP5')] == 'n/a'
+    assert data['status'][data['name'].index(injected_bad[0])] == 'bad'
 
     # check events.tsv is written
     events_tsv_fname = channels_tsv_name.replace('channels', 'events')
@@ -356,12 +374,13 @@ def test_vhdr():
 
     # Also cover iEEG
     # We use the same data and pretend that eeg channels are ecog
+    raw = mne.io.read_raw_brainvision(raw_fname)
     raw.set_channel_types({raw.ch_names[i]: 'ecog'
                            for i in mne.pick_types(raw.info, eeg=True)})
     output_path = _TempDir()
     write_raw_bids(raw, bids_basename, output_path, overwrite=False)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
 
@@ -382,13 +401,12 @@ def test_edf():
     raw.set_channel_types({'EMG': 'emg'})
 
     write_raw_bids(raw, bids_basename, output_path)
-    read_raw_bids(bids_basename + '_eeg.edf',
-                  output_path, return_events=False)
+    read_raw_bids(bids_basename + '_eeg.edf', output_path)
 
     bids_fname = bids_basename.replace('run-01', 'run-%s' % run2)
     write_raw_bids(raw, bids_fname, output_path, overwrite=True)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
     # ensure there is an EMG channel in the channels.tsv:
@@ -413,7 +431,7 @@ def test_edf():
     output_path = _TempDir()
     write_raw_bids(raw, bids_basename, output_path)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
 
@@ -426,11 +444,10 @@ def test_bdf():
     raw = mne.io.read_raw_edf(raw_fname)
     write_raw_bids(raw, bids_basename, output_path, overwrite=False)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
-    read_raw_bids(bids_basename + '_eeg.bdf',
-                  output_path, return_events=False)
+    read_raw_bids(bids_basename + '_eeg.bdf', output_path)
 
     raw.crop(0, raw.times[-2])
     with pytest.raises(AssertionError, match='cropped'):
@@ -459,14 +476,13 @@ def test_set():
 
     # proceed with the actual test for EEGLAB data
     write_raw_bids(raw, bids_basename, output_path, overwrite=False)
-    read_raw_bids(bids_basename + '_eeg.set',
-                  output_path, return_events=False)
+    read_raw_bids(bids_basename + '_eeg.set', output_path)
 
     with pytest.raises(FileExistsError, match="already exists"):  # noqa: F821
         write_raw_bids(raw, bids_basename, output_path=output_path,
                        overwrite=False)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)
 
     # check events.tsv is written
@@ -485,5 +501,5 @@ def test_set():
     output_path = _TempDir()
     write_raw_bids(raw, bids_basename, output_path)
 
-    cmd = ['bids-validator', output_path]
+    cmd = bids_validator_exe + [output_path]
     run_subprocess(cmd, shell=shell)