Repository: mne-tools/mne-python

Issue:
BrainVision Analyser data format version 2
Hello, 
I have some issues with importing Brainvision Files. When i do some preprocessing in Brainvision Analyser and then export the data, it exports data  headers in "Brain Vision Data Exchange Header File Version 2.0" format. It is slightly different from that, which is now implemented in MNE, so I had to brutally hack existing function to read my files. Unfortunately every update breaks my code and I don't feel up to the task of officially maintaining it on github. 

So I thought that it would be great if someone could review my code and maybe incorporate it into the MNE, you can find those dirty hacks here: http://pastebin.com/BbgsP4Wv It is for MNE 0.9

And some headers here:
[ICA_OBS_MRC_LTMAM2_01_p1.vhdr.txt](https://github.com/mne-tools/mne-python/files/34092/ICA_OBS_MRC_LTMAM2_01_p1.vhdr.txt)
[ICA_OBS_MRC_LTMAM2_01_p1.vmrk.txt](https://github.com/mne-tools/mne-python/files/34093/ICA_OBS_MRC_LTMAM2_01_p1.vmrk.txt)



Discussion:
the good approach is to send us the patch as a PR on github so we can integrate it and you don't have to maintain it.

please give it a try. And ask if you need help.

Should I fork and do a PULL request on 0.9 branch, or should I do it on master?

do it on master and we can backport to 0.10 eventually.


Resolving patch:
diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py
--- a/mne/io/brainvision/brainvision.py
+++ b/mne/io/brainvision/brainvision.py
@@ -54,6 +54,12 @@ class RawBrainVision(_BaseRaw):
         events (stimulus triggers will be unaffected). If None, response
         triggers will be ignored. Default is 0 for backwards compatibility, but
         typically another value or None will be necessary.
+    event_id : dict | None
+        The id of the event to consider. If None (default),
+        only stimulus events are added to the stimulus channel. If dict,
+        the keys will be mapped to trigger values on the stimulus channel
+        in addition to the stimulus events. Keys are case-sensitive.
+        Example: {'SyncStatus': 1; 'Pulse Artifact': 3}.
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
 
@@ -64,12 +70,14 @@ class RawBrainVision(_BaseRaw):
     @verbose
     def __init__(self, vhdr_fname, montage=None,
                  eog=('HEOGL', 'HEOGR', 'VEOGb'), misc=(),
-                 scale=1., preload=False, response_trig_shift=0, verbose=None):
+                 scale=1., preload=False, response_trig_shift=0,
+                 event_id=None, verbose=None):
         # Channel info and events
         logger.info('Extracting parameters from %s...' % vhdr_fname)
         vhdr_fname = os.path.abspath(vhdr_fname)
-        info, fmt, self._order, events = _get_vhdr_info(
-            vhdr_fname, eog, misc, response_trig_shift, scale)
+        info, fmt, self._order, mrk_fname, montage = _get_vhdr_info(
+            vhdr_fname, eog, misc, scale, montage)
+        events = _read_vmrk_events(mrk_fname, event_id, response_trig_shift)
         _check_update_montage(info, montage)
         with open(info['filename'], 'rb') as f:
             f.seek(0, os.SEEK_END)
@@ -139,13 +147,18 @@ def _create_event_ch(self, events, n_samp=None):
             self._data[-1] = self._event_ch
 
 
-def _read_vmrk_events(fname, response_trig_shift=0):
+def _read_vmrk_events(fname, event_id=None, response_trig_shift=0):
     """Read events from a vmrk file
 
     Parameters
     ----------
     fname : str
         vmrk file to be read.
+    event_id : dict | None
+        The id of the event to consider. If dict, the keys will be mapped to
+        trigger values on the stimulus channel. Example:
+        {'SyncStatus': 1; 'Pulse Artifact': 3}. If empty dict (default),
+        only stimulus events are added to the stimulus channel.
     response_trig_shift : int | None
         Integer to shift response triggers by. None ignores response triggers.
 
@@ -155,17 +168,14 @@ def _read_vmrk_events(fname, response_trig_shift=0):
         An array containing the whole recording's events, each row representing
         an event as (onset, duration, trigger) sequence.
     """
+    if event_id is None:
+        event_id = dict()
     # read vmrk file
     with open(fname, 'rb') as fid:
         txt = fid.read().decode('utf-8')
 
     header = txt.split('\n')[0].strip()
-    start_tag = 'Brain Vision Data Exchange Marker File'
-    if not header.startswith(start_tag):
-        raise ValueError("vmrk file should start with %r" % start_tag)
-    end_tag = 'Version 1.0'
-    if not header.endswith(end_tag):
-        raise ValueError("vmrk file should be %r" % end_tag)
+    _check_mrk_version(header)
     if (response_trig_shift is not None and
             not isinstance(response_trig_shift, int)):
         raise TypeError("response_trig_shift must be an integer or None")
@@ -184,16 +194,22 @@ def _read_vmrk_events(fname, response_trig_shift=0):
     events = []
     for info in items:
         mtype, mdesc, onset, duration = info.split(',')[:4]
+        onset = int(onset)
+        duration = int(duration)
         try:
             trigger = int(re.findall('[A-Za-z]*\s*?(\d+)', mdesc)[0])
-            if mdesc[0].lower() == 's' or response_trig_shift is not None:
-                if mdesc[0].lower() == 'r':
-                    trigger += response_trig_shift
-                onset = int(onset)
-                duration = int(duration)
-                events.append((onset, duration, trigger))
         except IndexError:
+            trigger = None
             pass
+        if mtype.lower().startswith('response'):
+            if response_trig_shift is not None:
+                trigger += response_trig_shift
+            else:
+                trigger = None
+        if mdesc in event_id:
+            trigger = event_id[mdesc]
+        if trigger:
+            events.append((onset, duration, trigger))
 
     events = np.array(events).reshape(-1, 3)
     return events
@@ -224,6 +240,24 @@ def _synthesize_stim_channel(events, n_samp):
     return stim_channel
 
 
+def _check_hdr_version(header):
+    tags = ['Brain Vision Data Exchange Header File Version 1.0',
+            'Brain Vision Data Exchange Header File Version 2.0']
+    if header not in tags:
+        raise ValueError("Currently only support %r, not %r"
+                         "Contact MNE-Developers for support."
+                         % (str(tags), header))
+
+
+def _check_mrk_version(header):
+    tags = ['Brain Vision Data Exchange Marker File, Version 1.0',
+            'Brain Vision Data Exchange Marker File, Version 2.0']
+    if header not in tags:
+        raise ValueError("Currently only support %r, not %r"
+                         "Contact MNE-Developers for support."
+                         % (str(tags), header))
+
+
 _orientation_dict = dict(MULTIPLEXED='F', VECTORIZED='C')
 _fmt_dict = dict(INT_16='short', INT_32='int', IEEE_FLOAT_32='single')
 _fmt_byte_dict = dict(short=2, int=4, single=4)
@@ -231,7 +265,7 @@ def _synthesize_stim_channel(events, n_samp):
 _unit_dict = {'V': 1., u'µV': 1e-6}
 
 
-def _get_vhdr_info(vhdr_fname, eog, misc, response_trig_shift, scale):
+def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
     """Extracts all the information from the header file.
 
     Parameters
@@ -244,12 +278,14 @@ def _get_vhdr_info(vhdr_fname, eog, misc, response_trig_shift, scale):
     misc : list of str
         Names of channels that should be designated MISC channels. Names
         should correspond to the electrodes in the vhdr file.
-    response_trig_shift : int | None
-        Integer to shift response triggers by. None ignores response triggers.
     scale : float
         The scaling factor for EEG data. Units are in volts. Default scale
         factor is 1.. For microvolts, the scale factor would be 1e-6. This is
         used when the header file does not specify the scale factor.
+    montage : str | True | None | instance of Montage
+        Path or instance of montage containing electrode positions.
+        If None, sensor locations are (0,0,0). See the documentation of
+        :func:`mne.channels.read_montage` for more information.
 
     Returns
     -------
@@ -270,8 +306,8 @@ def _get_vhdr_info(vhdr_fname, eog, misc, response_trig_shift, scale):
                       "not the '%s' file." % ext)
     with open(vhdr_fname, 'rb') as f:
         # extract the first section to resemble a cfg
-        l = f.readline().decode('utf-8').strip()
-        assert l == 'Brain Vision Data Exchange Header File Version 1.0'
+        header = f.readline().decode('utf-8').strip()
+        _check_hdr_version(header)
         settings = f.read().decode('utf-8')
 
     if settings.find('[Comment]') != -1:
@@ -308,18 +344,40 @@ def _get_vhdr_info(vhdr_fname, eog, misc, response_trig_shift, scale):
     cals = np.empty(info['nchan'])
     ranges = np.empty(info['nchan'])
     cals.fill(np.nan)
+    ch_dict = dict()
     for chan, props in cfg.items('Channel Infos'):
         n = int(re.findall(r'ch(\d+)', chan)[0]) - 1
         props = props.split(',')
         if len(props) < 4:
             props += ('V',)
         name, _, resolution, unit = props[:4]
+        ch_dict[chan] = name
         ch_names[n] = name
-        if resolution == "":  # For truncated vhdrs (e.g. EEGLAB export)
-            resolution = 0.000001
+        if resolution == "":
+            if not(unit):  # For truncated vhdrs (e.g. EEGLAB export)
+                resolution = 0.000001
+            else:
+                resolution = 1.  # for files with units specified, but not res
         unit = unit.replace(u'\xc2', u'')  # Remove unwanted control characters
         cals[n] = float(resolution)
         ranges[n] = _unit_dict.get(unit, unit) * scale
+
+    # create montage
+    if montage is True:
+        from ...transforms import _sphere_to_cartesian
+        from ...channels.montage import Montage
+        montage_pos = list()
+        montage_names = list()
+        for ch in cfg.items('Coordinates'):
+            montage_names.append(ch_dict[ch[0]])
+            radius, theta, phi = map(float, ch[1].split(','))
+            # 1: radius, 2: theta, 3: phi
+            pos = _sphere_to_cartesian(r=radius, theta=theta, phi=phi)
+            montage_pos.append(pos)
+        montage_sel = np.arange(len(montage_pos))
+        montage = Montage(montage_pos, montage_names, 'Brainvision',
+                          montage_sel)
+
     ch_names[-1] = 'STI 014'
     cals[-1] = 1.
     ranges[-1] = 1.
@@ -415,16 +473,15 @@ def _get_vhdr_info(vhdr_fname, eog, misc, response_trig_shift, scale):
             coord_frame=FIFF.FIFFV_COORD_HEAD))
 
     # for stim channel
-    marker_id = os.path.join(path, cfg.get('Common Infos', 'MarkerFile'))
-    events = _read_vmrk_events(marker_id, response_trig_shift)
+    mrk_fname = os.path.join(path, cfg.get('Common Infos', 'MarkerFile'))
     info._check_consistency()
-    return info, fmt, order, events
+    return info, fmt, order, mrk_fname, montage
 
 
 def read_raw_brainvision(vhdr_fname, montage=None,
                          eog=('HEOGL', 'HEOGR', 'VEOGb'), misc=(),
-                         scale=1., preload=False,
-                         response_trig_shift=0, verbose=None):
+                         scale=1., preload=False, response_trig_shift=0,
+                         event_id=None, verbose=None):
     """Reader for Brain Vision EEG file
 
     Parameters
@@ -455,6 +512,12 @@ def read_raw_brainvision(vhdr_fname, montage=None,
         events (stimulus triggers will be unaffected). If None, response
         triggers will be ignored. Default is 0 for backwards compatibility, but
         typically another value or None will be necessary.
+    event_id : dict | None
+        The id of the event to consider. If None (default),
+        only stimulus events are added to the stimulus channel. If dict,
+        the keys will be mapped to trigger values on the stimulus channel
+        in addition to the stimulus events. Keys are case-sensitive.
+        Example: {'SyncStatus': 1; 'Pulse Artifact': 3}.
     verbose : bool, str, int, or None
         If not None, override default verbose level (see mne.verbose).
 
@@ -469,6 +532,6 @@ def read_raw_brainvision(vhdr_fname, montage=None,
     """
     raw = RawBrainVision(vhdr_fname=vhdr_fname, montage=montage, eog=eog,
                          misc=misc, scale=scale,
-                         preload=preload, verbose=verbose,
+                         preload=preload, verbose=verbose, event_id=event_id,
                          response_trig_shift=response_trig_shift)
     return raw


Test patch:
diff --git a/mne/io/brainvision/tests/data/test.vmrk b/mne/io/brainvision/tests/data/test.vmrk
--- a/mne/io/brainvision/tests/data/test.vmrk
+++ b/mne/io/brainvision/tests/data/test.vmrk
@@ -21,3 +21,4 @@ Mk9=Stimulus,S255,4946,1,0
 Mk10=Response,R255,6000,1,0
 Mk11=Stimulus,S254,6620,1,0
 Mk12=Stimulus,S255,6630,1,0
+Mk13=SyncStatus,Sync On,7630,1,0
diff --git a/mne/io/brainvision/tests/data/testv2.vhdr b/mne/io/brainvision/tests/data/testv2.vhdr
new file mode 100644
--- /dev/null
+++ b/mne/io/brainvision/tests/data/testv2.vhdr
@@ -0,0 +1,107 @@
+Brain Vision Data Exchange Header File Version 2.0
+; Data created from history path: test/Raw Data
+
+[Common Infos]
+Codepage=UTF-8
+DataFile=test.eeg
+MarkerFile=testv2.vmrk
+DataFormat=BINARY
+; Data orientation: VECTORIZED=ch1,pt1, ch1,pt2..., MULTIPLEXED=ch1,pt1, ch2,pt1 ...
+DataOrientation=MULTIPLEXED
+DataType=TIMEDOMAIN
+NumberOfChannels=32
+DataPoints=7900
+; Sampling interval in microseconds if time domain (convert to Hertz:
+; 1000000 / SamplingInterval) or in Hertz if frequency domain:
+SamplingInterval=1000
+
+[User Infos]
+; Each entry: Prop<Number>=<Type>,<Name>,<Value>,<Value2>,...,<ValueN>
+; Property number must be unique. Types can be int, single, string, bool, byte, double, uint
+; or arrays of those, indicated int-array etc
+; Array types have more than one value, number of values determines size of array.
+; Fields are delimited by commas, commas in strings are written \1
+
+[Binary Infos]
+BinaryFormat=INT_16
+
+[Channel Infos]
+; Each entry: Ch<Channel number>=<Name>,<Reference channel name>,
+; <Resolution in "Unit">,<Unit>, Future extensions..
+; Fields are delimited by commas, some fields might be omitted (empty).
+; Commas in channel names are coded as "\1".
+Ch1=FP1,,0.5,µV
+Ch2=FP2,,0.5,µV
+Ch3=F3,,0.5,µV
+Ch4=F4,,0.5,µV
+Ch5=C3,,0.5,µV
+Ch6=C4,,0.5,µV
+Ch7=P3,,0.5,µV
+Ch8=P4,,0.5,µV
+Ch9=O1,,0.5,µV
+Ch10=O2,,0.5,µV
+Ch11=F7,,0.5,µV
+Ch12=F8,,0.5,µV
+Ch13=P7,,0.5,µV
+Ch14=P8,,0.5,µV
+Ch15=Fz,,0.5,µV
+Ch16=FCz,,0.5,µV
+Ch17=Cz,,0.5,µV
+Ch18=CPz,,0.5,µV
+Ch19=Pz,,0.5,µV
+Ch20=POz,,0.5,µV
+Ch21=FC1,,0.5,µV
+Ch22=FC2,,0.5,µV
+Ch23=CP1,,0.5,µV
+Ch24=CP2,,0.5,µV
+Ch25=FC5,,0.5,µV
+Ch26=FC6,,0.5,µV
+Ch27=CP5,,0.5,µV
+Ch28=CP6,,0.5,µV
+Ch29=HL,,0.5,µV
+Ch30=HR,,0.5,µV
+Ch31=Vb,,0.5,µV
+Ch32=ReRef,,0.5,µV
+
+[Channel User Infos]
+; Each entry: Prop<Number>=Ch<ChannelNumber>,<Type>,<Name>,<Value>,<Value2>,...,<ValueN>
+; Property number must be unique. Types can be int, single, string, bool, byte, double, uint
+; or arrays of those, indicated int-array etc
+; Array types have more than one value, number of values determines size of array.
+; Fields are delimited by commas, commas in strings are written \1
+; Properties are assigned to channels using their channel number.
+
+[Coordinates]
+; Each entry: Ch<Channel number>=<Radius>,<Theta>,<Phi>
+Ch1=1,-90,-72
+Ch2=1,90,72
+Ch3=1,-60,-51
+Ch4=1,60,51
+Ch5=1,-45,0
+Ch6=1,45,0
+Ch7=1,-60,51
+Ch8=1,60,-51
+Ch9=1,-90,72
+Ch10=1,90,-72
+Ch11=1,-90,-36
+Ch12=1,90,36
+Ch13=1,-90,36
+Ch14=1,90,-36
+Ch15=1,45,90
+Ch16=1,22,90
+Ch17=1,0,0
+Ch18=1,22,-90
+Ch19=1,45,-90
+Ch20=1,67,-90
+Ch21=1,-31,-46
+Ch22=1,31,46
+Ch23=1,-31,46
+Ch24=1,31,-46
+Ch25=1,-69,-21
+Ch26=1,69,21
+Ch27=1,-69,21
+Ch28=1,69,-21
+Ch29=0,0,0
+Ch30=0,0,0
+Ch31=0,0,0
+Ch32=0,0,0
diff --git a/mne/io/brainvision/tests/data/testv2.vmrk b/mne/io/brainvision/tests/data/testv2.vmrk
new file mode 100644
--- /dev/null
+++ b/mne/io/brainvision/tests/data/testv2.vmrk
@@ -0,0 +1,33 @@
+Brain Vision Data Exchange Marker File, Version 2.0
+; Data created from history path: test/Raw Data
+; The channel numbers are related to the channels in the exported file.
+
+[Common Infos]
+Codepage=UTF-8
+DataFile=test.eeg
+
+[Marker Infos]
+; Each entry: Mk<Marker number>=<Type>,<Description>,<Position in data points>,
+; <Size in data points>, <Channel number (0 = marker is related to all channels)>
+; Fields are delimited by commas, some fields might be omitted (empty).
+; Commas in type or description text are coded as "\1".
+Mk1=New Segment,,1,1,0,20131113161403794232
+Mk2=Stimulus,S253,487,1,0
+Mk3=Stimulus,S255,497,1,0
+Mk4=Stimulus,S254,1770,1,0
+Mk5=Stimulus,S255,1780,1,0
+Mk6=Stimulus,S254,3253,1,0
+Mk7=Stimulus,S255,3263,1,0
+Mk8=Stimulus,S253,4936,1,0
+Mk9=Stimulus,S255,4946,1,0
+Mk10=Response,R255,6000,1,0
+Mk11=Stimulus,S254,6620,1,0
+Mk12=Stimulus,S255,6630,1,0
+
+[Marker User Infos]
+; Each entry: Prop<Number>=Mk<Marker number>,<Type>,<Name>,<Value>,<Value2>,...,<ValueN>
+; Property number must be unique. Types can be int, single, string, bool, byte, double, uint
+; or arrays of those, indicated int-array etc
+; Array types have more than one value, number of values determines size of array.
+; Fields are delimited by commas, commas in strings are written \1
+; Properties are assigned to markers using their marker number.
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -23,6 +23,8 @@
 data_dir = op.join(op.dirname(op.abspath(FILE)), 'data')
 vhdr_path = op.join(data_dir, 'test.vhdr')
 vmrk_path = op.join(data_dir, 'test.vmrk')
+vhdr_v2_path = op.join(data_dir, 'testv2.vhdr')
+vmrk_v2_path = op.join(data_dir, 'testv2.vmrk')
 vhdr_highpass_path = op.join(data_dir, 'test_highpass.vhdr')
 montage = op.join(data_dir, 'test.hpts')
 eeg_bin = op.join(data_dir, 'test_bin_raw.fif')
@@ -76,6 +78,10 @@ def test_brainvision_data():
         else:
             raise RuntimeError("Unknown Channel: %s" % ch['ch_name'])
 
+    # test loading v2
+    read_raw_brainvision(vhdr_v2_path, eog=eog, preload=True,
+                         response_trig_shift=1000)
+
 
 def test_events():
     """Test reading and modifying events"""
@@ -128,6 +134,23 @@ def test_events():
                                 [4946, 1, 255],
                                 [6620, 1, 254],
                                 [6630, 1, 255]])
+    # check that events are read properly when event_id is specified for
+    # auxiliary events
+    raw = read_raw_brainvision(vhdr_path, eog=eog, preload=True,
+                               response_trig_shift=None,
+                               event_id={'Sync On': 5})
+    events = raw.get_brainvision_events()
+    assert_array_equal(events, [[487, 1, 253],
+                                [497, 1, 255],
+                                [1770, 1, 254],
+                                [1780, 1, 255],
+                                [3253, 1, 254],
+                                [3263, 1, 255],
+                                [4936, 1, 253],
+                                [4946, 1, 255],
+                                [6620, 1, 254],
+                                [6630, 1, 255],
+                                [7630, 1, 5]])
 
     assert_raises(TypeError, read_raw_brainvision, vhdr_path, eog=eog,
                   preload=True, response_trig_shift=0.1)