Repository: mne-tools/mne-python

Issue:
AssertionError brainvision.pyc file
Hello,

I can't open my .vhdr files for EEG analysis, there's got to be a pb with my brain vision.pyc file. Can someone guides me to fix this ?
Thxs in advance !
![screen shot 2016-11-05 at 19 56 47](https://cloud.githubusercontent.com/assets/23283446/20033075/8d84e58e-a398-11e6-9fe3-365863e911ec.png)



Discussion:
please share a problematic file.

how was this file created?

Thxs for your answer.
The file was downloaded with the brain vision package.

[brainvision.pyc.zip](https://github.com/mne-tools/mne-python/files/573966/brainvision.pyc.zip)

I meant share the .vhdr file so we can replicate the error and ideally fix
it.

thanks

ok, sure
[exemple.vhdr.zip](https://github.com/mne-tools/mne-python/files/574093/exemple.vhdr.zip)

ok so you have data in ASCII and our reader only supports data in binary
format
(binary by the way more compact on disk and avoids loss in numerical
precision)

can you save them to binary so it fixes your problem?

See https://github.com/mne-tools/mne-python/pull/3748

thxs for this quick solution.
do you have a method to access my .vhdr files content and translate it to binary ?

no.

but if you share a full dataset we can look into it.

you can't save again to binary with your brainvision software?

Would it be possible for you to get a short segment (say, 1 sec?) of the ASCII data and upload it? It might also be possible to add support for it at some point.

do you know which program other than my terminal can open vhdr file ? I can't open it with anything so can't get segments out of it. 
[Archive.zip](https://github.com/mne-tools/mne-python/files/578683/Archive.zip)
please find attached the vhdr + dat and vmrk corresponding files, if it can help.

thxs in advance


Resolving patch:
diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py
--- a/mne/io/brainvision/brainvision.py
+++ b/mne/io/brainvision/brainvision.py
@@ -19,9 +19,10 @@
 from ..constants import FIFF
 from ..meas_info import _empty_info
 from ..base import BaseRaw, _check_update_montage
-from ..utils import _read_segments_file, _synthesize_stim_channel
+from ..utils import (_read_segments_file, _synthesize_stim_channel,
+                     _mult_cal_one)
 
-from ...externals.six import StringIO
+from ...externals.six import StringIO, string_types
 from ...externals.six.moves import configparser
 
 
@@ -85,24 +86,45 @@ def __init__(self, vhdr_fname, montage=None,
         events = _read_vmrk_events(mrk_fname, event_id, response_trig_shift)
         _check_update_montage(info, montage)
         with open(info['filename'], 'rb') as f:
-            f.seek(0, os.SEEK_END)
-            n_samples = f.tell()
-        dtype_bytes = _fmt_byte_dict[fmt]
+            if isinstance(fmt, dict):  # ASCII, this will be slow :(
+                n_skip = 0
+                for ii in range(int(fmt['skiplines'])):
+                    n_skip += len(f.readline())
+                offsets = np.cumsum([n_skip] + [len(line) for line in f])
+                n_samples = len(offsets) - 1
+            else:
+                f.seek(0, os.SEEK_END)
+                n_samples = f.tell()
+                dtype_bytes = _fmt_byte_dict[fmt]
+                offsets = None
+                n_samples = n_samples // (dtype_bytes * (info['nchan'] - 1))
         self.preload = False  # so the event-setting works
-        last_samps = [(n_samples // (dtype_bytes * (info['nchan'] - 1))) - 1]
-        self._create_event_ch(events, last_samps[0] + 1)
+        self._create_event_ch(events, n_samples)
         super(RawBrainVision, self).__init__(
-            info, last_samps=last_samps, filenames=[info['filename']],
-            orig_format=fmt, preload=preload, verbose=verbose)
+            info, last_samps=[n_samples - 1], filenames=[info['filename']],
+            orig_format=fmt, preload=preload, verbose=verbose,
+            raw_extras=[offsets])
 
     def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         """Read a chunk of raw data."""
         # read data
-        dtype = _fmt_dtype_dict[self.orig_format]
-        n_data_ch = len(self.ch_names) - 1
-        _read_segments_file(self, data, idx, fi, start, stop, cals, mult,
-                            dtype=dtype, n_channels=n_data_ch,
-                            trigger_ch=self._event_ch)
+        if isinstance(self.orig_format, string_types):
+            dtype = _fmt_dtype_dict[self.orig_format]
+            n_data_ch = len(self.ch_names) - 1
+            _read_segments_file(self, data, idx, fi, start, stop, cals, mult,
+                                dtype=dtype, n_channels=n_data_ch,
+                                trigger_ch=self._event_ch)
+        else:
+            offsets = self._raw_extras[fi]
+            with open(self._filenames[fi], 'rb') as fid:
+                fid.seek(offsets[start])
+                block = np.empty((len(self.ch_names), stop - start))
+                for ii in range(stop - start):
+                    line = fid.readline().decode('ASCII')
+                    line = line.strip().replace(',', '.').split()
+                    block[:-1, ii] = list(map(float, line))
+            block[-1] = self._event_ch[start:stop]
+            _mult_cal_one(data, block, idx, cals, mult)
 
     def _get_brainvision_events(self):
         """Retrieve the events associated with the Brain Vision Raw object.
@@ -318,11 +340,10 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
         Events from the corresponding vmrk file.
     """
     scale = float(scale)
-
     ext = os.path.splitext(vhdr_fname)[-1]
     if ext != '.vhdr':
         raise IOError("The header file must be given to read the data, "
-                      "not the '%s' file." % ext)
+                      "not a file with extension '%s'." % ext)
     with open(vhdr_fname, 'rb') as f:
         # extract the first section to resemble a cfg
         header = f.readline()
@@ -364,21 +385,21 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale, montage):
     sfreq = 1e6 / cfg.getfloat('Common Infos', 'SamplingInterval')
     info = _empty_info(sfreq)
 
-    # check binary format
-    data_format = cfg.get('Common Infos', 'DataFormat')
-    if data_format != 'BINARY':
-        raise ValueError('Only data in binary format is supported. '
-                         'Your data are in %s.' % data_format)
     order = cfg.get('Common Infos', 'DataOrientation')
     if order not in _orientation_dict:
         raise NotImplementedError('Data Orientation %s is not supported'
                                   % order)
     order = _orientation_dict[order]
 
-    fmt = cfg.get('Binary Infos', 'BinaryFormat')
-    if fmt not in _fmt_dict:
-        raise NotImplementedError('Datatype %s is not supported' % fmt)
-    fmt = _fmt_dict[fmt]
+    data_format = cfg.get('Common Infos', 'DataFormat')
+    if data_format == 'BINARY':
+        fmt = cfg.get('Binary Infos', 'BinaryFormat')
+        if fmt not in _fmt_dict:
+            raise NotImplementedError('Datatype %s is not supported' % fmt)
+        fmt = _fmt_dict[fmt]
+    else:
+        fmt = dict((key, cfg.get('ASCII Infos', key))
+                   for key in cfg.options('ASCII Infos'))
 
     # load channel labels
     nchan = cfg.getint('Common Infos', 'NumberOfChannels') + 1


Test patch:
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -5,8 +5,9 @@
 #
 # License: BSD (3-clause)
 
-import os.path as op
 import inspect
+import os.path as op
+import shutil
 import warnings
 
 from nose.tools import assert_equal, assert_raises, assert_true
@@ -51,10 +52,51 @@
 montage = op.join(data_dir, 'test.hpts')
 eeg_bin = op.join(data_dir, 'test_bin_raw.fif')
 eog = ['HL', 'HR', 'Vb']
+event_id = {'Sync On': 5}
 
 warnings.simplefilter('always')
 
 
+def test_ascii():
+    """Test ASCII BV reading."""
+    raw = read_raw_brainvision(vhdr_path, event_id=event_id)
+    tempdir = _TempDir()
+    ascii_vhdr_path = op.join(tempdir, op.split(vhdr_path)[-1])
+    # copy marker file
+    shutil.copy(vhdr_path.replace('.vhdr', '.vmrk'),
+                ascii_vhdr_path.replace('.vhdr', '.vmrk'))
+    # modify header file
+    skipping = False
+    with open(ascii_vhdr_path, 'wb') as fout:
+        with open(vhdr_path, 'rb') as fin:
+            for line in fin:
+                # Common Infos section
+                if line.startswith(b'DataFormat'):
+                    line = b'DataFormat=ASCII\n'
+                elif line.startswith(b'DataFile='):
+                    line = b'DataFile=test.dat\n'
+                # Replace the "'Binary Infos'" section
+                elif line.startswith(b'[Binary Infos]'):
+                    skipping = True
+                    fout.write(b'[ASCII Infos]\nDecimalSymbol=.\nSkipLines=1\n'
+                               b'SkipColumns=0\n\n')
+                elif skipping and line.startswith(b'['):
+                    skipping = False
+                if not skipping:
+                    fout.write(line)
+    # create the .dat file
+    data, times = raw[:]
+    with open(ascii_vhdr_path.replace('.vhdr', '.dat'), 'wb') as fid:
+        fid.write(b' '.join(ch_name.encode('ASCII')
+                  for ch_name in raw.ch_names) + b'\n')
+        fid.write(b'\n'.join(b' '.join(b'%.3f' % dd for dd in d)
+                  for d in data[:-1].T / raw._cals[:-1]))
+    raw = read_raw_brainvision(ascii_vhdr_path, event_id=event_id)
+    data_new, times_new = raw[:]
+    assert_allclose(data_new, data, atol=1e-15)
+    assert_allclose(times_new, times)
+
+
 def test_brainvision_data_highpass_filters():
     """Test reading raw Brain Vision files with amplifier filter settings."""
     # Homogeneous highpass in seconds (default measurement unit)
@@ -71,37 +113,34 @@ def test_brainvision_data_highpass_filters():
     with warnings.catch_warnings(record=True) as w:  # event parsing
         raw = _test_raw_reader(
             read_raw_brainvision, vhdr_fname=vhdr_mixed_highpass_path,
-            montage=montage, eog=eog)
+            montage=montage, eog=eog, event_id=event_id)
 
-    trigger_warning = ['parse triggers that' in str(ww.message)
-                       for ww in w]
     lowpass_warning = ['different lowpass filters' in str(ww.message)
                        for ww in w]
     highpass_warning = ['different highpass filters' in str(ww.message)
                         for ww in w]
 
-    expected_warnings = zip(trigger_warning, lowpass_warning, highpass_warning)
+    expected_warnings = zip(lowpass_warning, highpass_warning)
 
-    assert_true(all(any([trg, lp, hp]) for trg, lp, hp in expected_warnings))
+    assert_true(all(any([lp, hp]) for lp, hp in expected_warnings))
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
 
     # Homogeneous highpass in Hertz
-    with warnings.catch_warnings(record=True) as w:  # event parsing
+    with warnings.catch_warnings(record=True):  # filter settings
         raw = _test_raw_reader(
             read_raw_brainvision, vhdr_fname=vhdr_highpass_hz_path,
-            montage=montage, eog=eog)
-    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
+            montage=montage, eog=eog, event_id=event_id)
 
     assert_equal(raw.info['highpass'], 10.)
     assert_equal(raw.info['lowpass'], 250.)
 
     # Heterogeneous highpass in Hertz
-    with warnings.catch_warnings(record=True) as w:  # event parsing
+    with warnings.catch_warnings(record=True):  # filter settings
         raw = _test_raw_reader(
             read_raw_brainvision, vhdr_fname=vhdr_mixed_highpass_hz_path,
-            montage=montage, eog=eog)
+            montage=montage, eog=eog, event_id=event_id)
 
     trigger_warning = ['parse triggers that' in str(ww.message)
                        for ww in w]
@@ -119,14 +158,12 @@ def test_brainvision_data_highpass_filters():
 
 
 def test_brainvision_data_lowpass_filters():
-    """Test reading raw Brain Vision files with amplifier LP filter settings"""
+    """Test files with amplifier LP filter settings."""
 
     # Homogeneous lowpass in Hertz (default measurement unit)
-    with warnings.catch_warnings(record=True) as w:  # event parsing
-        raw = _test_raw_reader(
-            read_raw_brainvision, vhdr_fname=vhdr_lowpass_path,
-            montage=montage, eog=eog)
-    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
+    raw = _test_raw_reader(
+        read_raw_brainvision, vhdr_fname=vhdr_lowpass_path,
+        montage=montage, eog=eog, event_id=event_id)
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
@@ -135,57 +172,49 @@ def test_brainvision_data_lowpass_filters():
     with warnings.catch_warnings(record=True) as w:  # event parsing
         raw = _test_raw_reader(
             read_raw_brainvision, vhdr_fname=vhdr_mixed_lowpass_path,
-            montage=montage, eog=eog)
+            montage=montage, eog=eog, event_id=event_id)
 
-    trigger_warning = ['parse triggers that' in str(ww.message)
-                       for ww in w]
     lowpass_warning = ['different lowpass filters' in str(ww.message)
                        for ww in w]
     highpass_warning = ['different highpass filters' in str(ww.message)
                         for ww in w]
 
-    expected_warnings = zip(trigger_warning, lowpass_warning, highpass_warning)
+    expected_warnings = zip(lowpass_warning, highpass_warning)
 
-    assert_true(all(any([trg, lp, hp]) for trg, lp, hp in expected_warnings))
+    assert_true(all(any([lp, hp]) for lp, hp in expected_warnings))
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
 
     # Homogeneous lowpass in seconds
-    with warnings.catch_warnings(record=True) as w:  # event parsing
-        raw = _test_raw_reader(
-            read_raw_brainvision, vhdr_fname=vhdr_lowpass_s_path,
-            montage=montage, eog=eog)
-    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
+    raw = _test_raw_reader(
+        read_raw_brainvision, vhdr_fname=vhdr_lowpass_s_path,
+        montage=montage, eog=eog, event_id=event_id)
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
 
     # Heterogeneous lowpass in seconds
-    with warnings.catch_warnings(record=True) as w:  # event parsing
+    with warnings.catch_warnings(record=True) as w:  # filter settings
         raw = _test_raw_reader(
             read_raw_brainvision, vhdr_fname=vhdr_mixed_lowpass_s_path,
-            montage=montage, eog=eog)
+            montage=montage, eog=eog, event_id=event_id)
 
-    trigger_warning = ['parse triggers that' in str(ww.message)
-                       for ww in w]
     lowpass_warning = ['different lowpass filters' in str(ww.message)
                        for ww in w]
     highpass_warning = ['different highpass filters' in str(ww.message)
                         for ww in w]
 
-    expected_warnings = zip(trigger_warning, lowpass_warning, highpass_warning)
+    expected_warnings = zip(lowpass_warning, highpass_warning)
 
-    assert_true(all(any([trg, lp, hp]) for trg, lp, hp in expected_warnings))
+    assert_true(all(any([lp, hp]) for lp, hp in expected_warnings))
 
     assert_equal(raw.info['highpass'], 0.1)
     assert_equal(raw.info['lowpass'], 250.)
 
 
 def test_brainvision_data_partially_disabled_hw_filters():
-    """Test reading raw Brain Vision files with heterogeneous amplifier
-       filter settings including non-numeric values
-    """
+    """Test heterogeneous filter settings including non-numeric values."""
     with warnings.catch_warnings(record=True) as w:  # event parsing
         raw = _test_raw_reader(
             read_raw_brainvision,
@@ -225,11 +254,9 @@ def test_brainvision_data():
     assert_raises(ValueError, read_raw_brainvision, vhdr_path, montage,
                   preload=True, scale="foo")
 
-    with warnings.catch_warnings(record=True) as w:  # event parsing
-        raw_py = _test_raw_reader(
-            read_raw_brainvision, vhdr_fname=vhdr_path, montage=montage,
-            eog=eog, misc='auto')
-    assert_true(all('parse triggers that' in str(ww.message) for ww in w))
+    raw_py = _test_raw_reader(
+        read_raw_brainvision, vhdr_fname=vhdr_path, montage=montage,
+        eog=eog, misc='auto', event_id=event_id)
 
     assert_true('RawBrainVision' in repr(raw_py))
 
@@ -275,82 +302,75 @@ def test_events():
     tempdir = _TempDir()
 
     # check that events are read and stim channel is synthesized correcly
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
-        raw = read_raw_brainvision(vhdr_path, eog=eog, preload=True)
-        events = raw._get_brainvision_events()
-        assert_array_equal(events, [[487, 1, 253],
-                                    [497, 1, 255],
-                                    [1770, 1, 254],
-                                    [1780, 1, 255],
-                                    [3253, 1, 254],
-                                    [3263, 1, 255],
-                                    [4936, 1, 253],
-                                    [4946, 1, 255],
-                                    [6000, 1, 255],
-                                    [6620, 1, 254],
-                                    [6630, 1, 255]])
-        assert_equal(len(w), 1)  # for dropping Sync & R255 events
+    raw = read_raw_brainvision(vhdr_path, eog=eog, event_id=event_id)
+    events = raw._get_brainvision_events()
+    events = events[events[:, 2] != event_id['Sync On']]
+    assert_array_equal(events, [[487, 1, 253],
+                                [497, 1, 255],
+                                [1770, 1, 254],
+                                [1780, 1, 255],
+                                [3253, 1, 254],
+                                [3263, 1, 255],
+                                [4936, 1, 253],
+                                [4946, 1, 255],
+                                [6000, 1, 255],
+                                [6620, 1, 254],
+                                [6630, 1, 255]])
 
     # check that events are read and stim channel is synthesized correcly and
     # response triggers are shifted like they're supposed to be.
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
-        raw = read_raw_brainvision(vhdr_path, eog=eog, preload=True,
-                                   response_trig_shift=1000)
-        events = raw._get_brainvision_events()
-        assert_array_equal(events, [[487, 1, 253],
-                                    [497, 1, 255],
-                                    [1770, 1, 254],
-                                    [1780, 1, 255],
-                                    [3253, 1, 254],
-                                    [3263, 1, 255],
-                                    [4936, 1, 253],
-                                    [4946, 1, 255],
-                                    [6000, 1, 1255],
-                                    [6620, 1, 254],
-                                    [6630, 1, 255]])
-        assert_equal(len(w), 1)  # for dropping Sync & R255 events
+    raw = read_raw_brainvision(vhdr_path, eog=eog,
+                               response_trig_shift=1000, event_id=event_id)
+    events = raw._get_brainvision_events()
+    events = events[events[:, 2] != event_id['Sync On']]
+    assert_array_equal(events, [[487, 1, 253],
+                                [497, 1, 255],
+                                [1770, 1, 254],
+                                [1780, 1, 255],
+                                [3253, 1, 254],
+                                [3263, 1, 255],
+                                [4936, 1, 253],
+                                [4946, 1, 255],
+                                [6000, 1, 1255],
+                                [6620, 1, 254],
+                                [6630, 1, 255]])
 
     # check that events are read and stim channel is synthesized correcly and
     # response triggers are ignored.
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
-        raw = read_raw_brainvision(vhdr_path, eog=eog, preload=True,
+    with warnings.catch_warnings(record=True):  # ignored events
+        raw = read_raw_brainvision(vhdr_path, eog=eog,
                                    response_trig_shift=None)
-        events = raw._get_brainvision_events()
-        assert_array_equal(events, [[487, 1, 253],
-                                    [497, 1, 255],
-                                    [1770, 1, 254],
-                                    [1780, 1, 255],
-                                    [3253, 1, 254],
-                                    [3263, 1, 255],
-                                    [4936, 1, 253],
-                                    [4946, 1, 255],
-                                    [6620, 1, 254],
-                                    [6630, 1, 255]])
-        assert_equal(len(w), 1)  # for dropping Sync & R255 events
+    events = raw._get_brainvision_events()
+    events = events[events[:, 2] != event_id['Sync On']]
+    assert_array_equal(events, [[487, 1, 253],
+                                [497, 1, 255],
+                                [1770, 1, 254],
+                                [1780, 1, 255],
+                                [3253, 1, 254],
+                                [3263, 1, 255],
+                                [4936, 1, 253],
+                                [4946, 1, 255],
+                                [6620, 1, 254],
+                                [6630, 1, 255]])
 
     # check that events are read properly when event_id is specified for
     # auxiliary events
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
+    with warnings.catch_warnings(record=True):  # dropped events
         raw = read_raw_brainvision(vhdr_path, eog=eog, preload=True,
                                    response_trig_shift=None,
-                                   event_id={'Sync On': 5})
-        events = raw._get_brainvision_events()
-        assert_array_equal(events, [[487, 1, 253],
-                                    [497, 1, 255],
-                                    [1770, 1, 254],
-                                    [1780, 1, 255],
-                                    [3253, 1, 254],
-                                    [3263, 1, 255],
-                                    [4936, 1, 253],
-                                    [4946, 1, 255],
-                                    [6620, 1, 254],
-                                    [6630, 1, 255],
-                                    [7630, 1, 5]])
-        assert_equal(len(w), 1)  # parsing Sync event, missing R255
+                                   event_id=event_id)
+    events = raw._get_brainvision_events()
+    assert_array_equal(events, [[487, 1, 253],
+                                [497, 1, 255],
+                                [1770, 1, 254],
+                                [1780, 1, 255],
+                                [3253, 1, 254],
+                                [3263, 1, 255],
+                                [4936, 1, 253],
+                                [4946, 1, 255],
+                                [6620, 1, 254],
+                                [6630, 1, 255],
+                                [7630, 1, 5]])
 
     assert_raises(TypeError, read_raw_brainvision, vhdr_path, eog=eog,
                   preload=True, response_trig_shift=0.1)
@@ -358,15 +378,10 @@ def test_events():
                   preload=True, response_trig_shift=np.nan)
 
     # Test that both response_trig_shit and event_id can be set
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter('always')
-        read_raw_brainvision(vhdr_path, eog=eog, preload=False,
-                             response_trig_shift=100,
-                             event_id={'Sync On': 5})
-
-        mne_events = find_events(raw, stim_channel='STI 014')
-        assert_array_equal(events[:, [0, 2]], mne_events[:, [0, 2]])
-        assert_equal(len(w), 0)  # parsing the Sync event
+    read_raw_brainvision(vhdr_path, eog=eog, preload=False,
+                         response_trig_shift=100, event_id=event_id)
+    mne_events = find_events(raw, stim_channel='STI 014')
+    assert_array_equal(events[:, [0, 2]], mne_events[:, [0, 2]])
 
     # modify events and check that stim channel is updated
     index = events[:, 2] == 255