Repository: mne-tools/mne-python

Issue:
WIP, ENH: BrainVision impedance parsing
I was using the MNE tool for parsing BrainVision data files in a paper where we needed to parse all the information from the segmentation and impedance sections of a header file. As i did not find these informations in the current raw parsed data, an enhancement of the current parser was implemented that adds these sections to the raw info.

Considering a possible wider use case for these changes, we decided that this enhancement could be included in the tool directly.

---
**Enhancements:**

The function `parse_segmentation` takes any of the key-value pairs in the segmentation subsections of the BrainVision file starting with `S e g m e n t a t i o n  /  A v e r a g i n g`, as well as some of the added key-values in the Common Infos. The segmentation is stored in a new info item `segmentation`.

The function `parse_impedance` creates a `dict` of all the channel impedances from the header file. All of the channels in the raw info are iterated over, and if an impedance record exists, they are appended to the channel dict.

The header file that these were tested on is in `mne/io/brainvision/tests/data/test_segmentation_impedance.vhdr` (also used for the unit tests)

---

**Problems:**

However, even though these changes work well in our local environment and the parsing of the values in itself works good, implementing these changes directly to MNE causes some of the already existing unit tests to fail, and I'm not really sure how to solve these issues. When i try to add any new attributes to the info structure, even an empty list.

These are the failing tests:
```
FAILED mne/io/brainvision/tests/test_brainvision.py::test_brainvision_data_highpass_filters - RuntimeWarning: Could not get size for self.info
FAILED mne/io/brainvision/tests/test_brainvision.py::test_brainvision_data_lowpass_filters - RuntimeWarning: Could not get size for self.info
FAILED mne/io/brainvision/tests/test_brainvision.py::test_brainvision_data_partially_disabled_hw_filters - AssertionError: assert {'acq_pars', ... 'comps', ...} == {'acq_pars', ... 'comps', ...}
FAILED mne/io/brainvision/tests/test_brainvision.py::test_brainvision_data_software_filters_latin1_global_units - AssertionError: assert {'acq_pars', ... 'comps', ...} == {'acq_pars', ... 'comps', ...}
FAILED mne/io/brainvision/tests/test_brainvision.py::test_brainvision_data - RuntimeWarning: Could not get size for self.info
```

Any ideas where the problem could be for the existing tests and how this could be improved?



Discussion:

Resolving patch:
diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py
--- a/mne/io/brainvision/brainvision.py
+++ b/mne/io/brainvision/brainvision.py
@@ -51,6 +51,11 @@ class RawBrainVision(BaseRaw):
     %(preload)s
     %(verbose)s
 
+    Attributes
+    ----------
+    impedances : dict
+        A dictionary of all electrodes and their impedances.
+
     See Also
     --------
     mne.io.Raw : Documentation of attribute and methods.
@@ -93,6 +98,11 @@ def __init__(self, vhdr_fname,
 
         self.set_montage(montage)
 
+        settings, cfg, cinfo, _ = _aux_vhdr_info(vhdr_fname)
+        split_settings = settings.splitlines()
+        self.impedances = _parse_impedance(split_settings,
+                                           self.info['meas_date'])
+
         # Get annotations from vmrk file
         annots = read_annotations(mrk_fname, info['sfreq'])
         self.set_annotations(annots)
@@ -843,3 +853,101 @@ def _check_bv_annot(descriptions):
     bv_markers = (set(_BV_EVENT_IO_OFFSETS.keys())
                   .union(set(_OTHER_ACCEPTED_MARKERS.keys())))
     return len(markers_basename - bv_markers) == 0
+
+
+def _parse_impedance(settings, recording_date=None):
+    """Parse impedances from the header file.
+
+    Parameters
+    ----------
+    settings : list
+        The header settings lines fom the VHDR file.
+    recording_date : datetime.datetime | None
+        The date of the recording as extracted from the VMRK file.
+
+    Returns
+    -------
+    impedances : dict
+        A dictionary of all electrodes and their impedances.
+    """
+    ranges = _parse_impedance_ranges(settings)
+    impedance_setting_lines = [i for i in settings if
+                               i.startswith('Impedance [') and
+                               i.endswith(' :')]
+    impedances = dict()
+    if len(impedance_setting_lines) > 0:
+        idx = settings.index(impedance_setting_lines[0])
+        impedance_setting = impedance_setting_lines[0].split()
+        impedance_unit = impedance_setting[1].lstrip('[').rstrip(']')
+        impedance_time = None
+
+        # If we have a recording date, we can update it with the time of
+        # impedance measurement
+        if recording_date is not None:
+            meas_time = [int(i) for i in impedance_setting[3].split(':')]
+            impedance_time = recording_date.replace(hour=meas_time[0],
+                                                    minute=meas_time[1],
+                                                    second=meas_time[2],
+                                                    microsecond=0)
+        for setting in settings[idx + 1:]:
+            # Parse channel impedances until we find a line that doesn't start
+            # with a channel name and optional +/- polarity for passive elecs
+            match = re.match(r'[ a-zA-Z0-9_+-]+:', setting)
+            if match:
+                channel_name = match.group().rstrip(':')
+                channel_imp_line = setting.split()
+                imp_as_number = re.findall(r"[-+]?\d*\.\d+|\d+",
+                                           channel_imp_line[-1])
+                channel_impedance = dict(
+                    imp=float(imp_as_number[0]) if imp_as_number else np.nan,
+                    imp_unit=impedance_unit,
+                )
+                if impedance_time is not None:
+                    channel_impedance.update({'imp_meas_time': impedance_time})
+
+                if channel_name == 'Ref' and 'Reference' in ranges:
+                    channel_impedance.update(ranges['Reference'])
+                elif channel_name == 'Gnd' and 'Ground' in ranges:
+                    channel_impedance.update(ranges['Ground'])
+                elif 'Data' in ranges:
+                    channel_impedance.update(ranges['Data'])
+                impedances[channel_name] = channel_impedance
+            else:
+                break
+    return impedances
+
+
+def _parse_impedance_ranges(settings):
+    """Parse the selected electrode impedance ranges from the header.
+
+    Parameters
+    ----------
+    settings : list
+        The header settings lines fom the VHDR file.
+
+    Returns
+    -------
+    electrode_imp_ranges : dict
+        A dictionary of impedance ranges for each type of electrode.
+    """
+    impedance_ranges = [item for item in settings if
+                        "Selected Impedance Measurement Range" in item]
+    electrode_imp_ranges = dict()
+    if impedance_ranges:
+        if len(impedance_ranges) == 1:
+            img_range = impedance_ranges[0].split()
+            for electrode_type in ['Data', 'Reference', 'Ground']:
+                electrode_imp_ranges[electrode_type] = {
+                    "imp_lower_bound": float(img_range[-4]),
+                    "imp_upper_bound": float(img_range[-2]),
+                    "imp_range_unit": img_range[-1]
+                }
+        else:
+            for electrode_range in impedance_ranges:
+                electrode_range = electrode_range.split()
+                electrode_imp_ranges[electrode_range[0]] = {
+                    "imp_lower_bound": float(electrode_range[6]),
+                    "imp_upper_bound": float(electrode_range[8]),
+                    "imp_range_unit": electrode_range[9]
+                }
+    return electrode_imp_ranges
diff --git a/mne/utils/numerics.py b/mne/utils/numerics.py
--- a/mne/utils/numerics.py
+++ b/mne/utils/numerics.py
@@ -758,7 +758,10 @@ def object_diff(a, b, pre=''):
         else:
             for ii, (xx1, xx2) in enumerate(zip(a, b)):
                 out += object_diff(xx1, xx2, pre + '[%s]' % ii)
-    elif isinstance(a, (str, int, float, bytes, np.generic)):
+    elif isinstance(a, float):
+        if not _array_equal_nan(a, b):
+            out += pre + ' value mismatch (%s, %s)\n' % (a, b)
+    elif isinstance(a, (str, int, bytes, np.generic)):
         if a != b:
             out += pre + ' value mismatch (%s, %s)\n' % (a, b)
     elif a is None:
diff --git a/tutorials/io/plot_20_reading_eeg_data.py b/tutorials/io/plot_20_reading_eeg_data.py
--- a/tutorials/io/plot_20_reading_eeg_data.py
+++ b/tutorials/io/plot_20_reading_eeg_data.py
@@ -39,13 +39,15 @@
 `Brain Products website <https://www.brainproducts.com/productdetails.php?id=21&tab=5>`_
 
 BrainVision EEG files can be read in using :func:`mne.io.read_raw_brainvision`
-with the .vhdr header file as an input.
+with the ``.vhdr`` header file as an input.
 
 .. warning:: Renaming BrainVision files can be problematic due to their
              multifile structure. See this
-             `example <https://mne.tools/mne-bids/stable/auto_examples/rename_brainvision_files.html>`_
+             `example <https://mne.tools/mne-bids/stable/auto_examples/rename_brainvision_files.html#sphx-glr-auto-examples-rename-brainvision-files-py>`_
              for an instruction.
 
+.. note:: For *writing* BrainVision files, you can use the Python package
+          `pybv <https://pypi.org/project/pybv/>`_.
 
 .. _import-edf:
 


Test patch:
diff --git a/mne/io/brainvision/tests/data/test_highpass.vhdr b/mne/io/brainvision/tests/data/test_highpass.vhdr
--- a/mne/io/brainvision/tests/data/test_highpass.vhdr
+++ b/mne/io/brainvision/tests/data/test_highpass.vhdr
@@ -101,10 +101,7 @@ S o f t w a r e  F i l t e r s
 ==============================
 Disabled
 
-
-Data Electrodes Selected Impedance Measurement Range: 0 - 100 kOhm
-Ground Electrode Selected Impedance Measurement Range: 0 - 10 kOhm
-Reference Electrode Selected Impedance Measurement Range: 0 - 10 kOhm
+Data / Ref / Gnd Electrodes Selected Impedance Measurement Range: 0 - 100 kOhm
 Impedance [kOhm] at 16:12:27 :
 FP1:        ???
 FP2:        ???
diff --git a/mne/io/brainvision/tests/data/test_mixed_lowpass.vhdr b/mne/io/brainvision/tests/data/test_mixed_lowpass.vhdr
--- a/mne/io/brainvision/tests/data/test_mixed_lowpass.vhdr
+++ b/mne/io/brainvision/tests/data/test_mixed_lowpass.vhdr
@@ -48,9 +48,9 @@ Ch25=FC5,,0.5,µV
 Ch26=FC6,,0.5,µV
 Ch27=CP5,,0.5,µV
 Ch28=CP 6,,0.5,µV
-Ch29=HL,,0.5,µV
-Ch30=HR,,0.5,µV
-Ch31=Vb,,0.5,µV
+Ch29=ECG,,0.5,µV
+Ch30=HEOG,,0.5,µV
+Ch31=VEOG,,0.5,µV
 Ch32=ReRef,,0.5,µV
 
 [Comment]
@@ -92,12 +92,55 @@ Channels
 26    FC6         26               0.5 µV             10              250              Off                0
 27    CP5         27               0.5 µV             10              250              Off                0
 28    CP 6        28               0.5 µV             10              250              Off                0
-29    HL          29               0.5 µV             10              250              Off                0
-30    HR          30               0.5 µV             10              250              Off                0
-31    Vb          31               0.5 µV             10              250              Off                0
+29    ECG         29               0.5 µV             10              250              Off                0
+30    HEOG        30               0.5 µV             10              250              Off                0
+31    VEOG        31               0.5 µV             10              250              Off                0
 32    ReRef       32               0.5 µV             10              125              Off                0
 
 S o f t w a r e  F i l t e r s
 ==============================
 Disabled
 
+
+
+Impedances Imported from actiCAP Control Software:
+Impedance [kOhm] at 10:17:02 :
+FP1:        ???
+FP2:        ???
+F3:         ???
+F4:         ???
+C3:         ???
+C4:         ???
+P3:         ???
+P4:         ???
+O1:         ???
+O2:         ???
+F7:         ???
+F8:         ???
+P7:         ???
+P8:         ???
+Fz:         ???
+FCz:        ???
+Cz:         ???
+CPz:        ???
+Pz:         ???
+POz:        ???
+FC1:        ???
+FC2:        ???
+CP1:        ???
+CP2:        ???
+FC5:        ???
+FC6:        ???
+CP5:        ???
+CP 6:       ???
+ECG+:        35
+ECG-:        46
+HEOG+:        6
+HEOG-:        8
+VEOG+:        3
+VEOG-:        4
+ReRef:        0
+Ref:          8
+Gnd:          2.5
+
+; this is a comment
diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py
--- a/mne/io/brainvision/tests/test_brainvision.py
+++ b/mne/io/brainvision/tests/test_brainvision.py
@@ -12,7 +12,8 @@
                            assert_allclose, assert_equal)
 import pytest
 
-from mne.utils import run_tests_if_main, _stamp_to_dt
+import datetime
+from mne.utils import run_tests_if_main, _stamp_to_dt, object_diff
 from mne import pick_types, read_annotations, concatenate_raws
 from mne.io.constants import FIFF
 from mne.io import read_raw_fif, read_raw_brainvision
@@ -650,4 +651,57 @@ def test_event_id_stability_when_save_and_fif_reload(tmpdir):
     assert_array_equal(events, original_events)
 
 
+def test_parse_impedance():
+    """Test case for parsing the impedances from header."""
+    expected_imp_meas_time = datetime.datetime(2013, 11, 13, 16, 12, 27,
+                                               tzinfo=datetime.timezone.utc)
+    expected_imp_unit = 'kOhm'
+    expected_electrodes = [
+        'FP1', 'FP2', 'F3', 'F4', 'C3', 'C4', 'P3', 'P4', 'O1', 'O2', 'F7',
+        'F8', 'P7', 'P8', 'Fz', 'FCz', 'Cz', 'CPz', 'Pz', 'POz', 'FC1', 'FC2',
+        'CP1', 'CP2', 'FC5', 'FC6', 'CP5', 'CP6', 'HL', 'HR', 'Vb', 'ReRef',
+        'Ref', 'Gnd'
+    ]
+    n_electrodes = len(expected_electrodes)
+    expected_imps = [np.nan] * (n_electrodes - 2) + [0., 4.]
+    expected_imp_lower_bound = 0.
+    expected_imp_upper_bound = [100.] * (n_electrodes - 2) + [10., 10.]
+
+    expected_impedances = {elec: {
+        'imp': expected_imps[i],
+        'imp_unit': expected_imp_unit,
+        'imp_meas_time': expected_imp_meas_time,
+        'imp_lower_bound': expected_imp_lower_bound,
+        'imp_upper_bound': expected_imp_upper_bound[i],
+        'imp_range_unit': expected_imp_unit,
+    } for i, elec in enumerate(expected_electrodes)}
+
+    raw = read_raw_brainvision(vhdr_path, eog=eog)
+    assert object_diff(expected_impedances, raw.impedances) == ''
+
+    # Test "Impedances Imported from actiCAP Control Software"
+    expected_imp_meas_time = expected_imp_meas_time.replace(hour=10,
+                                                            minute=17,
+                                                            second=2)
+    tmpidx = expected_electrodes.index('CP6')
+    expected_electrodes = expected_electrodes[:tmpidx] + [
+        'CP 6', 'ECG+', 'ECG-', 'HEOG+', 'HEOG-', 'VEOG+', 'VEOG-', 'ReRef',
+        'Ref', 'Gnd'
+    ]
+    n_electrodes = len(expected_electrodes)
+    expected_imps = [np.nan] * (n_electrodes - 9) + [
+        35., 46., 6., 8., 3., 4., 0., 8., 2.5
+    ]
+    expected_impedances = {elec: {
+        'imp': expected_imps[i],
+        'imp_unit': expected_imp_unit,
+        'imp_meas_time': expected_imp_meas_time,
+    } for i, elec in enumerate(expected_electrodes)}
+
+    with pytest.warns(RuntimeWarning, match='different .*pass filters'):
+        raw = read_raw_brainvision(vhdr_mixed_lowpass_path,
+                                   eog=['HEOG', 'VEOG'], misc=['ECG'])
+    assert object_diff(expected_impedances, raw.impedances) == ''
+
+
 run_tests_if_main()
diff --git a/mne/utils/tests/test_numerics.py b/mne/utils/tests/test_numerics.py
--- a/mne/utils/tests/test_numerics.py
+++ b/mne/utils/tests/test_numerics.py
@@ -310,6 +310,8 @@ def test_object_diff_with_nan():
 
     assert object_diff(d0, d1) == ''
     assert object_diff(d0, d2) != ''
+    assert object_diff(np.nan, np.nan) == ''
+    assert object_diff(np.nan, 3.5) == ' value mismatch (nan, 3.5)\n'
 
 
 def test_hash():