Repository: mne-tools/mne-python

Issue:
CTF data loading with non-zero compensation grade error
I was trying to load CTF data. I only put the .ds dir as the first parameter, all other parameters are default value. Then meet a runtime error that "RuntimeError: Reference magnetometer channel with non-zero compensation grade 1".

the code and the results are in the below

Example:

```Python
%matplotlib tk
import mne
import matplotlib.pyplot as plt
import os.path as op
import os
import glob
import numpy as np

sub = "A1922"
data_path = op.join('/Users/yaocongd/local/Data/MEEG_2018/',sub);
dir_list = os.listdir(data_path)
mne.io.read_raw_ctf(op.join(data_path,dir_list[2]), system_clock='truncate', preload=True, clean_names=False, verbose=None)
```


#### Actual results

```
ds directory : /Users/yaocongd/local/Data/MEEG_2018/A1922/A1922_GlasgowFace_20190619_05.ds
    res4 data read.
    hc data read.
    Separate EEG position data file not present.
    Quaternion matching (desired vs. transformed):
       4.47   82.95    0.00 mm <->    4.47   82.95    0.00 mm (orig :  -67.80   49.75 -257.65 mm) diff =    0.000 mm
      -4.47  -82.95    0.00 mm <->   -4.47  -82.95    0.00 mm (orig :   44.38  -72.48 -266.51 mm) diff =    0.000 mm
     104.26    0.00    0.00 mm <->  104.26    0.00    0.00 mm (orig :   60.64   61.29 -243.12 mm) diff =    0.000 mm
    Coordinate transformations established.
    Polhemus data for 3 HPI coils added
    Device coordinate locations for 3 HPI coils added
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-11-015f3654086c> in <module>
      2 data_path = op.join('/Users/yaocongd/local/Data/MEEG_2018/',sub);
      3 dir_list = os.listdir(data_path)
----> 4 mne.io.read_raw_ctf(op.join(data_path,dir_list[2]), system_clock='truncate', preload=True, clean_names=False, verbose=None)

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/io/ctf/ctf.py in read_raw_ctf(directory, system_clock, preload, clean_names, verbose)
     62     """
     63     return RawCTF(directory, system_clock, preload=preload,
---> 64                   clean_names=clean_names, verbose=verbose)
     65 
     66 

</Users/yaocongd/anaconda3/envs/mne/lib/python3.6/site-packages/mne/externals/decorator.py:decorator-gen-150> in __init__(self, directory, system_clock, preload, verbose, clean_names)

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/utils/_logging.py in wrapper(*args, **kwargs)
     87             with use_log_level(verbose_level):
     88                 return function(*args, **kwargs)
---> 89         return function(*args, **kwargs)
     90     return FunctionMaker.create(
     91         function, 'return decfunc(%(signature)s)',

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/io/ctf/ctf.py in __init__(self, directory, system_clock, preload, verbose, clean_names)
    115 
    116         # Compose a structure which makes fiff writing a piece of cake
--> 117         info = _compose_meas_info(res4, coils, coord_trans, eeg)
    118         info['dig'] += digs
    119         info['bads'] += _read_bad_chans(directory, info)

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/io/ctf/info.py in _compose_meas_info(res4, coils, trans, eeg)
    442         if trans['t_ctf_head_head'] is not None:
    443             info['ctf_head_t'] = trans['t_ctf_head_head']
--> 444     info['chs'] = _convert_channel_info(res4, trans, eeg is None)
    445     info['comps'] = _convert_comp_data(res4)
    446     if eeg is None:

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/io/ctf/info.py in _convert_channel_info(res4, t, use_eeg_pos)
    219             if cch['sensor_type_index'] == CTF.CTFV_REF_MAG_CH:
    220                 ch['kind'] = FIFF.FIFFV_REF_MEG_CH
--> 221                 _check_comp_ch(cch, 'Reference magnetometer')
    222                 ch['coil_type'] = FIFF.FIFFV_COIL_CTF_REF_MAG
    223                 nref += 1

~/anaconda3/envs/mne/lib/python3.6/site-packages/mne/io/ctf/info.py in _check_comp_ch(cch, kind, desired)
    138         if cch['grad_order_no'] != 0:
    139             raise RuntimeError('%s channel with non-zero compensation grade %s'
--> 140                                % (kind, cch['grad_order_no']))
    141     else:
    142         if desired is None:

RuntimeError: Reference magnetometer channel with non-zero compensation grade 1
```



Discussion:
Can you upload this dataset somewhere? It's weird that there is a reference magnetometer channel that has had gradient compensation applied to it.

Do you know where this dataset was acquired, and with which CTF acquisition software version?
Eric this issue comes from one of our attendees in Gasgow. We can try to take this offline if data sharing would turn out to be difficult. Perhaps we should also try reading it with MNE-C.
+1 to try with mne-c

>

Thank you all very much. This data has both MEG and EEG recorded, I guess the problem is might because the EEG channel. Let me check the validation of data with fieldtrip first tomorrow.
Hi! I got the same error with a mixed EEG/MEG dataset. Maybe the CTF system uses the eeg as a ref and thus all magnetometers are compensated, even the REF ones.
@bloyl as our CTF expert do you have any thought on this? do we some tiny dataset to replicate this?
I agree that it seems odd that a ref magnetometer would have gradient compensation applied.
I've never seen this issue on our MEG only data and unfortunately I don't have any datasets that have both MEG and EEG so I can't replicate the issue.

Is there a dataset available?
Hi! I have the same error with a dataset containing only MEG data.
Here you can find the dataset:
https://drive.google.com/file/d/1s3Ofe4RCkV5xp-UG-EdNVLtp3c6o8z6h/view?usp=sharing

I can replicate the problem too. I have no clue why we impose a compensation grade of 0 for reference channels.
in the shared dataset there are 27 reference channels with a compensation grade of 1 and makes our reader crash.

@bloyl or @larsoner any idea?
I didn't think the reference channels changed when you applied gradient
compensation. So I'm not sure how/why the reference channels get marked as
having compensation applied.  Our data that is acquired with 3rd order
compensation turned on doesn't have this problem. Maybe if I turn it on/off
on processing I could get a dataset in this state.

Nichalas, is this raw ctf data or has it been processed at all? If so what
software did you use to do the processing.

I'll try this week to open this dataset using ctf's software and see what
happens.



On Sun, Jul 5, 2020, 10:26 AM Alexandre Gramfort <notifications@github.com>
wrote:

> I can replicate the problem too. I have no clue why we impose a
> compensation grade of 0 for reference channels.
> in the shared dataset there are 27 reference channels with a compensation
> grade of 1 and makes our reader crash.
>
> @bloyl <https://github.com/bloyl> or @larsoner
> <https://github.com/larsoner> any idea?
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/mne-tools/mne-python/issues/6488#issuecomment-653895431>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ABKTXHL5M4XGWYSOD3JJXITR2CEP5ANCNFSM4H3JZ4SA>
> .
>

We tried to apply an offline 3rd order compensation and the error remained.
Then, we did a test measurement with 1st and 3rd order (data attached) and the 3rd order dataset could be successfully imported. 

dataset: https://drive.google.com/file/d/16yZ0fdT24i3lK3D6jvgpn70VWBmZXFG1/view?usp=sharing   
so you have a pb with data recorded using 1st order without any
processing applied.

If so we should fix this. Can you read the files easily with
Brainstorm or Fieldtrip?

Yes, I think this is the case. 
I didn't have any issue loading the data with Fieldtrip (haven't tried with Brainstorm).
I added this back in #3440. Perhaps we can try disabling the check and check to see if forwards computed in both cases match...?

Resolving patch:
diff --git a/mne/io/ctf/ctf.py b/mne/io/ctf/ctf.py
--- a/mne/io/ctf/ctf.py
+++ b/mne/io/ctf/ctf.py
@@ -11,7 +11,8 @@
 import numpy as np
 
 from .._digitization import _format_dig_points
-from ...utils import verbose, logger, _clean_names, fill_doc, _check_option
+from ...utils import (verbose, logger, _clean_names, fill_doc, _check_option,
+                      _validate_type)
 
 from ..base import BaseRaw
 from ..utils import _mult_cal_one, _blk_read_lims
@@ -90,9 +91,11 @@ class RawCTF(BaseRaw):
     def __init__(self, directory, system_clock='truncate', preload=False,
                  verbose=None, clean_names=False):  # noqa: D102
         # adapted from mne_ctf2fiff.c
-        if not isinstance(directory, str) or \
-                not directory.endswith('.ds'):
-            raise TypeError('directory must be a directory ending with ".ds"')
+        _validate_type(directory, 'path-like', 'directory')
+        directory = str(directory)
+        if not directory.endswith('.ds'):
+            raise TypeError('directory must be a directory ending with ".ds", '
+                            f'got {directory}')
         if not op.isdir(directory):
             raise ValueError('directory does not exist: "%s"' % directory)
         _check_option('system_clock', system_clock, ['ignore', 'truncate'])
diff --git a/mne/io/ctf/info.py b/mne/io/ctf/info.py
--- a/mne/io/ctf/info.py
+++ b/mne/io/ctf/info.py
@@ -134,17 +134,12 @@ def _at_origin(x):
 
 
 def _check_comp_ch(cch, kind, desired=None):
-    if 'reference' in kind.lower():
-        if cch['grad_order_no'] != 0:
-            raise RuntimeError('%s channel with non-zero compensation grade %s'
-                               % (kind, cch['grad_order_no']))
-    else:
-        if desired is None:
-            desired = cch['grad_order_no']
-        if cch['grad_order_no'] != desired:
-            raise RuntimeError('%s channel with inconsistent compensation '
-                               'grade %s, should be %s'
-                               % (kind, cch['grad_order_no'], desired))
+    if desired is None:
+        desired = cch['grad_order_no']
+    if cch['grad_order_no'] != desired:
+        raise RuntimeError('%s channel with inconsistent compensation '
+                           'grade %s, should be %s'
+                           % (kind, cch['grad_order_no'], desired))
     return desired
 
 
@@ -218,17 +213,14 @@ def _convert_channel_info(res4, t, use_eeg_pos):
             # Set the coil type
             if cch['sensor_type_index'] == CTF.CTFV_REF_MAG_CH:
                 ch['kind'] = FIFF.FIFFV_REF_MEG_CH
-                _check_comp_ch(cch, 'Reference magnetometer')
                 ch['coil_type'] = FIFF.FIFFV_COIL_CTF_REF_MAG
                 nref += 1
                 ch['logno'] = nref
             elif cch['sensor_type_index'] == CTF.CTFV_REF_GRAD_CH:
                 ch['kind'] = FIFF.FIFFV_REF_MEG_CH
                 if off_diag:
-                    _check_comp_ch(cch, 'Reference off-diagonal gradiometer')
                     ch['coil_type'] = FIFF.FIFFV_COIL_CTF_OFFDIAG_REF_GRAD
                 else:
-                    _check_comp_ch(cch, 'Reference gradiometer')
                     ch['coil_type'] = FIFF.FIFFV_COIL_CTF_REF_GRAD
                 nref += 1
                 ch['logno'] = nref


Test patch:
diff --git a/mne/io/ctf/tests/test_ctf.py b/mne/io/ctf/tests/test_ctf.py
--- a/mne/io/ctf/tests/test_ctf.py
+++ b/mne/io/ctf/tests/test_ctf.py
@@ -12,11 +12,13 @@
 from numpy.testing import assert_allclose, assert_array_equal
 import pytest
 
+import mne
 from mne import (pick_types, read_annotations, create_info,
-                 events_from_annotations)
+                 events_from_annotations, make_forward_solution)
 from mne.transforms import apply_trans
 from mne.io import read_raw_fif, read_raw_ctf, RawArray
 from mne.io.compensator import get_current_comp
+from mne.io.ctf.constants import CTF
 from mne.io.tests.test_raw import _test_raw_reader
 from mne.tests.test_annotations import _assert_annotations_equal
 from mne.utils import (run_tests_if_main, _clean_names, catch_logging,
@@ -383,4 +385,45 @@ def test_read_ctf_annotations_smoke_test():
     _assert_annotations_equal(raw.annotations, annot, 1e-6)
 
 
+def _read_res4_mag_comp(dsdir):
+    res = mne.io.ctf.res4._read_res4(dsdir)
+    for ch in res['chs']:
+        if ch['sensor_type_index'] == CTF.CTFV_REF_MAG_CH:
+            ch['grad_order_no'] = 1
+    return res
+
+
+def _bad_res4_grad_comp(dsdir):
+    res = mne.io.ctf.res4._read_res4(dsdir)
+    for ch in res['chs']:
+        if ch['sensor_type_index'] == CTF.CTFV_MEG_CH:
+            ch['grad_order_no'] = 1
+            break
+    return res
+
+
+@testing.requires_testing_data
+def test_read_ctf_mag_bad_comp(tmpdir, monkeypatch):
+    """Test CTF reader with mag comps and bad comps."""
+    path = op.join(ctf_dir, ctf_fname_continuous)
+    raw_orig = read_raw_ctf(path)
+    assert raw_orig.compensation_grade == 0
+    monkeypatch.setattr(mne.io.ctf.ctf, '_read_res4', _read_res4_mag_comp)
+    raw_mag_comp = read_raw_ctf(path)
+    assert raw_mag_comp.compensation_grade == 0
+    sphere = mne.make_sphere_model()
+    src = mne.setup_volume_source_space(pos=50., exclude=5., bem=sphere)
+    assert src[0]['nuse'] == 26
+    for grade in (0, 1):
+        raw_orig.apply_gradient_compensation(grade)
+        raw_mag_comp.apply_gradient_compensation(grade)
+        args = (None, src, sphere, True, False)
+        fwd_orig = make_forward_solution(raw_orig.info, *args)
+        fwd_mag_comp = make_forward_solution(raw_mag_comp.info, *args)
+        assert_allclose(fwd_orig['sol']['data'], fwd_mag_comp['sol']['data'])
+    monkeypatch.setattr(mne.io.ctf.ctf, '_read_res4', _bad_res4_grad_comp)
+    with pytest.raises(RuntimeError, match='inconsistent compensation grade'):
+        read_raw_ctf(path)
+
+
 run_tests_if_main()