Repository: mne-tools/mne-python

Issue:
Elekta xfit dipole file reader
@Eric89GXL it looks like the parsing of the xfit dipole file is prone to being botched likely due to how users decide to write out the `.dip` file (i.e. what fields to include). [Here](https://www.dropbox.com/s/hk3ez4512ml1168/All_157_RH.dip.txt?dl=0) is an e.g. dipole file; and the resulting `__dict__`

```
{'amplitude': array([  4.36000000e-08]),
 'gof': array([-5.8]),
 'name': None,
 'ori': array([[ 0.4587156 ,  0.16055046,  0.40825688]]),
 'pos': array([[ 0.0382,  0.0056,  0.0674]]),
 'times': array([ 0.363])}
```

`gof` and `amplitude` fields are off and I am not sure how the `ori` is being represented here.



Discussion:
I guess we'll have to modify the reading and writing functions to actually
deal with the header text :(


Resolving patch:
diff --git a/mne/dipole.py b/mne/dipole.py
--- a/mne/dipole.py
+++ b/mne/dipole.py
@@ -386,29 +386,80 @@ def read_dipole(fname, verbose=None):
     _check_fname(fname, overwrite=True, must_exist=True)
     if fname.endswith('.fif') or fname.endswith('.fif.gz'):
         return _read_dipole_fixed(fname)
-    try:
-        data = np.loadtxt(fname, comments='%')
-    except:
-        data = np.loadtxt(fname, comments='#')  # handle 2 types of comments...
-    name = None
+    else:
+        return _read_dipole_text(fname)
+
+
+def _read_dipole_text(fname):
+    """Read a dipole text file."""
+    # Figure out the special fields
+    need_header = True
+    def_line = name = None
+    # There is a bug in older np.loadtxt regarding skipping fields,
+    # so just read the data ourselves (need to get name and header anyway)
+    data = list()
     with open(fname, 'r') as fid:
-        for line in fid.readlines():
-            if line.startswith('##') or line.startswith('%%'):
-                m = re.search('Name "(.*) dipoles"', line)
-                if m:
-                    name = m.group(1)
-                    break
-    if data.ndim == 1:
-        data = data[None, :]
+        for line in fid:
+            if not (line.startswith('%') or line.startswith('#')):
+                need_header = False
+                data.append(line.strip().split())
+            else:
+                if need_header:
+                    def_line = line
+                if line.startswith('##') or line.startswith('%%'):
+                    m = re.search('Name "(.*) dipoles"', line)
+                    if m:
+                        name = m.group(1)
+        del line
+    data = np.atleast_2d(np.array(data, float))
+    if def_line is None:
+        raise IOError('Dipole text file is missing field definition '
+                      'comment, cannot parse %s' % (fname,))
+    # actually parse the fields
+    def_line = def_line.lstrip('%').lstrip('#').strip()
+    # MNE writes it out differently than Elekta, let's standardize them...
+    fields = re.sub('([X|Y|Z] )\(mm\)',  # "X (mm)", etc.
+                    lambda match: match.group(1).strip() + '/mm', def_line)
+    fields = re.sub('\((.*?)\)',  # "Q(nAm)", etc.
+                    lambda match: '/' + match.group(1), fields)
+    fields = re.sub('(begin|end) ',  # "begin" and "end" with no units
+                    lambda match: match.group(1) + '/ms', fields)
+    fields = fields.lower().split()
+    used_fields = ('begin/ms',
+                   'x/mm', 'y/mm', 'z/mm',
+                   'q/nam',
+                   'qx/nam', 'qy/nam', 'qz/nam',
+                   'g/%')
+    missing_fields = sorted(set(used_fields) - set(fields))
+    if len(missing_fields) > 0:
+        raise RuntimeError('Could not find necessary fields in header: %s'
+                           % (missing_fields,))
+    ignored_fields = sorted(set(fields) - set(used_fields) - set(['end/ms']))
+    if len(ignored_fields) > 0:
+        warn('Ignoring extra fields in dipole file: %s' % (ignored_fields,))
+    if len(fields) != data.shape[1]:
+        raise IOError('More data fields (%s) found than data columns (%s): %s'
+                      % (len(fields), data.shape[1], fields))
+
     logger.info("%d dipole(s) found" % len(data))
-    times = data[:, 0] / 1000.
-    pos = 1e-3 * data[:, 2:5]  # put data in meters
-    amplitude = data[:, 5]
+
+    if 'end/ms' in fields:
+        if np.diff(data[:, [fields.index('begin/ms'),
+                            fields.index('end/ms')]], 1, -1).any():
+            warn('begin and end fields differed, but only begin will be used '
+                 'to store time values')
+
+    # Find the correct column in our data array, then scale to proper units
+    idx = [fields.index(field) for field in used_fields]
+    assert len(idx) == 9
+    times = data[:, idx[0]] / 1000.
+    pos = 1e-3 * data[:, idx[1:4]]  # put data in meters
+    amplitude = data[:, idx[4]]
     norm = amplitude.copy()
     amplitude /= 1e9
     norm[norm == 0] = 1
-    ori = data[:, 6:9] / norm[:, np.newaxis]
-    gof = data[:, 9]
+    ori = data[:, idx[5:8]] / norm[:, np.newaxis]
+    gof = data[:, idx[8]]
     return Dipole(times, pos, amplitude, ori, gof, name)
 
 


Test patch:
diff --git a/mne/tests/test_dipole.py b/mne/tests/test_dipole.py
--- a/mne/tests/test_dipole.py
+++ b/mne/tests/test_dipole.py
@@ -41,6 +41,8 @@
 fname_fwd = op.join(data_path, 'MEG', 'sample',
                     'sample_audvis_trunc-meg-eeg-oct-6-fwd.fif')
 fname_xfit_dip = op.join(data_path, 'dip', 'fixed_auto.fif')
+fname_xfit_dip_txt = op.join(data_path, 'dip', 'fixed_auto.dip')
+fname_xfit_seq_txt = op.join(data_path, 'dip', 'sequential.dip')
 fname_ctf = op.join(data_path, 'CTF', 'testdata_ctf_short.ds')
 subjects_dir = op.join(data_path, 'subjects')
 
@@ -331,6 +333,14 @@ def test_dipole_fixed():
     """Test reading a fixed-position dipole (from Xfit)"""
     dip = read_dipole(fname_xfit_dip)
     _check_roundtrip_fixed(dip)
+    with warnings.catch_warnings(record=True) as w:  # unused fields
+        dip_txt = read_dipole(fname_xfit_dip_txt)
+    assert_true(any('extra fields' in str(ww.message) for ww in w))
+    assert_allclose(dip.info['chs'][0]['loc'][:3], dip_txt.pos[0])
+    assert_allclose(dip_txt.amplitude[0], 12.1e-9)
+    with warnings.catch_warnings(record=True):  # unused fields
+        dip_txt_seq = read_dipole(fname_xfit_seq_txt)
+    assert_allclose(dip_txt_seq.gof, [27.3, 46.4, 43.7, 41., 37.3, 32.5])
 
 
 def _check_roundtrip_fixed(dip):