Repository: mne-tools/mne-python

Issue:
Bug read_raw_brainvision
from mailing list

```
I am using mne.io.read_raw_brainvision to read bipolar EMG and EKG data saved as brainvision files (vhdr).
I have 4 channels: EMGleft,EMGright,EKG and STI 014 with the triggers.

However, when I import the data in MNE python, the first three channels are not correctly imported.
They all look like a combination of EMG data, however they do not correspond to the original data, particularly even the third channel is not at all EKG. 
You can see attached to the email the images with a portion of imported data from the three channels.

Is it possible that the function is using a reference by default? I don't see this option in the function.
This is how I am using the function:

emg = mne.io.read_raw_brainvision(emgFname,montage=None, misc='auto',scale=1.0,preload=True, verbose=True)
```

answer by @bburle 

```
Investigating the issue, it appears that mne.io.read_raw_brainvision is
currently not using the information about data orientation in the header
(.vhdr) file:

DataOrientation=VECTORIZED

Apparently, the function assumes that data are multiplexed. It's just a
matter of transposing the matrix while reading. I'll try to pinpoint the
lines in mne.io.read_raw_brainvision where this should be done.
```

problematic file is here

[EMG.zip](https://github.com/mne-tools/mne-python/files/1028311/EMG.zip)



Discussion:
Being not familiar with the "_BaseRaw" object, I haven't (yet) identified where the problem is. But to advance a bit, here is a short piece of code that 1) replicate the problem, 2) open the file correctly (through direct memmap). I haven't been able to get the same (wrong) data as mne through a direct memmap, by transposing and/or reshaping differently the array. Hope this can be useful...


```
import numpy as np
import pylab as plt
import mne

emgFname = 'sb1b1'

mne_data = mne.io.read_raw_brainvision(emgFname+'.vhdr',montage=None, misc='auto',scale=1.0,preload=True, verbose=True)
plt.subplot(311)
plt.plot(mne_data._data[0,22000:120000]) #portion of data where something interesting occurs
plt.subplot(312)
plt.plot(mne_data._data[1,22000:120000])
plt.subplot(313)
plt.plot(mne_data._data[1,22000:120000])
#wrong data

raw_data = np.memmap(emgFname+'.eeg', np.float32)
rs_data = raw_data.reshape((3,813056)) #hard coded based on the vhdr file
plt.subplot(311)
plt.plot(rs_data[0,22000:120000])
plt.subplot(312)
plt.plot(rs_data[1,22000:120000])
plt.subplot(313)
plt.plot(rs_data[2,22000:120000])
# data OK

```
Is this on master? IIRC we had a problem with long channel names earlier that should now be fixed. WIll try to look at it later, but might take a while :(

@teonbrooks 

Resolving patch:
diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py
--- a/mne/io/brainvision/brainvision.py
+++ b/mne/io/brainvision/brainvision.py
@@ -81,8 +81,10 @@ def __init__(self, vhdr_fname, montage=None,
         # Channel info and events
         logger.info('Extracting parameters from %s...' % vhdr_fname)
         vhdr_fname = os.path.abspath(vhdr_fname)
-        info, data_filename, fmt, self._order, mrk_fname, montage = \
+        info, data_filename, fmt, order, mrk_fname, montage, n_samples = \
             _get_vhdr_info(vhdr_fname, eog, misc, scale, montage)
+        self._order = order
+        self._n_samples = n_samples
         events = _read_vmrk_events(mrk_fname, event_id, response_trig_shift)
         _check_update_montage(info, montage)
         with open(data_filename, 'rb') as f:
@@ -108,7 +110,9 @@ def __init__(self, vhdr_fname, montage=None,
     def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         """Read a chunk of raw data."""
         # read data
-        if isinstance(self.orig_format, string_types):
+        if self._order == 'C':
+            _read_segments_c(self, data, idx, fi, start, stop, cals, mult)
+        elif isinstance(self.orig_format, string_types):
             dtype = _fmt_dtype_dict[self.orig_format]
             n_data_ch = len(self.ch_names) - 1
             _read_segments_file(self, data, idx, fi, start, stop, cals, mult,
@@ -162,6 +166,26 @@ def _create_event_ch(self, events, n_samp=None):
             self._data[-1] = self._event_ch
 
 
+def _read_segments_c(raw, data, idx, fi, start, stop, cals, mult):
+    """Read chunk of vectorized raw data."""
+    n_samples = raw._n_samples
+    dtype = _fmt_dtype_dict[raw.orig_format]
+    n_bytes = _fmt_byte_dict[raw.orig_format]
+    n_channels = len(raw.ch_names)
+    trigger_ch = raw._event_ch
+    block = np.zeros((n_channels, stop - start))
+    with open(raw._filenames[fi], 'rb', buffering=0) as fid:
+        for ch_id in np.arange(n_channels)[idx]:
+            if ch_id == n_channels - 1:  # stim channel
+                stim_ch = trigger_ch[start:stop]
+                block[ch_id] = stim_ch
+                continue
+            fid.seek(start * n_bytes + ch_id * n_bytes * n_samples)
+            block[ch_id] = np.fromfile(fid, dtype, stop - start)
+
+        _mult_cal_one(data, block, idx, cals, mult)
+
+
 def _read_vmrk_events(fname, event_id=None, response_trig_shift=0):
     """Read events from a vmrk file.
 
@@ -402,8 +426,26 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
         fmt = dict((key, cfg.get('ASCII Infos', key))
                    for key in cfg.options('ASCII Infos'))
 
+    # locate EEG and marker files
+    path = os.path.dirname(vhdr_fname)
+    data_filename = os.path.join(path, cfg.get('Common Infos', 'DataFile'))
+    info['meas_date'] = int(time.time())
+    info['buffer_size_sec'] = 1.  # reasonable default
+
     # load channel labels
     nchan = cfg.getint('Common Infos', 'NumberOfChannels') + 1
+    n_samples = None
+    if order == 'C':
+        try:
+            n_samples = cfg.getint('Common Infos', 'DataPoints')
+        except configparser.NoOptionError:
+            logger.warning('No info on DataPoints found. Inferring number of '
+                           'samples from the data file size.')
+            with open(data_filename, 'rb') as fid:
+                fid.seek(0, 2)
+                n_bytes = fid.tell()
+                n_samples = n_bytes // _fmt_byte_dict[fmt] // (nchan - 1)
+
     ch_names = [''] * nchan
     cals = np.empty(nchan)
     ranges = np.empty(nchan)
@@ -622,12 +664,6 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
                      'Highest (weakest) filter setting (%0.2f Hz%s) '
                      'will be stored.' % (info['lowpass'], nyquist))
 
-    # locate EEG and marker files
-    path = os.path.dirname(vhdr_fname)
-    data_filename = os.path.join(path, cfg.get('Common Infos', 'DataFile'))
-    info['meas_date'] = int(time.time())
-    info['buffer_size_sec'] = 1.  # reasonable default
-
     # Creates a list of dicts of eeg channels for raw.info
     logger.info('Setting channel info structure...')
     info['chs'] = []
@@ -661,7 +697,7 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
     mrk_fname = os.path.join(path, cfg.get('Common Infos', 'MarkerFile'))
     info._update_redundant()
     info._check_consistency()
-    return info, data_filename, fmt, order, mrk_fname, montage
+    return info, data_filename, fmt, order, mrk_fname, montage, n_samples
 
 
 def read_raw_brainvision(vhdr_fname, montage=None,


Test patch:
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -297,6 +297,47 @@ def test_brainvision_data():
                          response_trig_shift=1000)
 
 
+def test_brainvision_vectorized_data():
+    """Test reading BrainVision data files with vectorized data."""
+
+    raw = read_raw_brainvision(vhdr_old_path, preload=True)
+
+    assert_array_equal(raw._data.shape, (30, 251))
+
+    first_two_samples_all_chs = np.array([[+5.22000008e-06, +5.10000000e-06],
+                                          [+2.10000000e-06, +2.27000008e-06],
+                                          [+1.15000000e-06, +1.33000002e-06],
+                                          [+4.00000000e-07, +4.00000000e-07],
+                                          [-3.02999992e-06, -2.82000008e-06],
+                                          [+2.71000004e-06, +2.45000000e-06],
+                                          [+2.41000004e-06, +2.36000004e-06],
+                                          [+1.01999998e-06, +1.18000002e-06],
+                                          [-1.33999996e-06, -1.25000000e-06],
+                                          [-2.60000000e-06, -2.46000004e-06],
+                                          [+6.80000019e-07, +8.00000000e-07],
+                                          [+1.48000002e-06, +1.48999996e-06],
+                                          [+1.61000004e-06, +1.51000004e-06],
+                                          [+7.19999981e-07, +8.60000038e-07],
+                                          [-3.00000000e-07, -4.00000006e-08],
+                                          [-1.20000005e-07, +6.00000024e-08],
+                                          [+8.19999981e-07, +9.89999962e-07],
+                                          [+1.13000002e-06, +1.28000002e-06],
+                                          [+1.08000002e-06, +1.33999996e-06],
+                                          [+2.20000005e-07, +5.69999981e-07],
+                                          [-4.09999990e-07, +4.00000006e-08],
+                                          [+5.19999981e-07, +9.39999962e-07],
+                                          [+1.01000004e-06, +1.51999998e-06],
+                                          [+1.01000004e-06, +1.55000000e-06],
+                                          [-1.43000002e-06, -1.13999996e-06],
+                                          [+3.65000000e-06, +3.65999985e-06],
+                                          [+4.15999985e-06, +3.79000015e-06],
+                                          [+9.26999969e-06, +8.95999985e-06],
+                                          [-7.35999985e-06, -7.18000031e-06],
+                                          [+0.00000000e+00, +0.00000000e+00]])
+
+    assert_array_almost_equal(raw._data[:, :2], first_two_samples_all_chs)
+
+
 def test_events():
     """Test reading and modifying events."""
     tempdir = _TempDir()