Repository: mne-tools/mne-python

Issue:
ctf2fif conversion problem
Hi guys, 
I am just starting to use mne-python on CTF data. I used the C tool mne_ctf2fif to convert the data ('mne_ctf2fif --ds data.ds/ --fif data01')
I have a .ds folder with the .meg4 file (ctf) of 1.11go and the resulting .fif file is only around 200mo. When I looked at the events, there is indeed more than three quarter of events missing. Nevertheless, it seems to work with smaller files (around 300mo). Is there a size limit? I am running under Apple el capitan on a macbookpro.
Thanks a lot!



Discussion:
Do you mean the file is 1.11 GB and the resulting FIF file is 200 MB? I'm not sure what you mean by "mo" and "go" units...?

FIF does have a size limit, but it can be overcome by writing split files. I'm not sure if the C tools will do this automatically, but mne-python should now that we also have CTF reading capabilities.

The C tools do not do the splitting.

> On Dec 14, 2015, at 4:22 PM, Eric Larson notifications@github.com wrote:
> 
> Do you mean the file is 1.11 GB and the resulting FIF file is 200 MB? I'm not sure what you mean by "mo" and "go" units...?
> 
> FIF does have a size limit, but it can be overcome by writing split files. I'm not sure if the C tools will do this automatically, but mne-python should now that we also have CTF reading capabilities.
> 
> —
> Reply to this email directly or view it on GitHub https://github.com/mne-tools/mne-python/issues/2698#issuecomment-164449834.

The information in this e-mail is intended only for the person to whom it is
addressed. If you believe this e-mail was sent to you in error and the e-mail
contains patient information, please contact the Partners Compliance HelpLine at
http://www.partners.org/complianceline . If the e-mail was sent to you in error
but does not contain patient information, please contact the sender and properly
dispose of the e-mail.

Ahh, thanks @mshamalainen. @romquentin can you try using mne-python to read the raw file? You need to install latest `master` and then do something like:

```
raw = mne.io.read_raw_ctf(directory_name)
raw.save('test_raw.fif')
```

If I remember correctly, the CTF raw data files also fit in the 2GB limitation, maybe a few samples omitted from the end.

> On Dec 14, 2015, at 4:27 PM, Eric Larson notifications@github.com wrote:
> 
> Ahh, thanks @mshamalainen https://github.com/mshamalainen. @romquentin https://github.com/romquentin can you try using mne-python to read the raw file? You need to install latest master and then do something like:
> 
> raw = mne.io.read_raw_ctf(directory_name)
> raw.save('test_raw.fif')
> —
> Reply to this email directly or view it on GitHub https://github.com/mne-tools/mne-python/issues/2698#issuecomment-164450856.

The information in this e-mail is intended only for the person to whom it is
addressed. If you believe this e-mail was sent to you in error and the e-mail
contains patient information, please contact the Partners Compliance HelpLine at
http://www.partners.org/complianceline . If the e-mail was sent to you in error
but does not contain patient information, please contact the sender and properly
dispose of the e-mail.

Thank you for your quick response. Yes, I was talking about GB sorry. 
I tried mne-python to read the file with this code:

import mne
from mne.io import Raw
raw=mne.io.read_raw_ctf('rom_wmtask_20151203_01.ds')
raw.save('test_raw.fif')

## and I got this error:

ValueError                                Traceback (most recent call last)
<ipython-input-12-f1b3ff64d4c3> in <module>()
----> 1 raw.save('test_raw.fif')

/Users/romain/Downloads/src/mne/mne/io/base.py in save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, verbose)

/Users/romain/Downloads/src/mne/mne/utils.py in verbose(function, _args, *_kwargs)
    549         finally:
    550             set_log_level(old_level)
--> 551     return function(_args, *_kwargs)
    552 
    553 

/Users/romain/Downloads/src/mne/mne/io/base.py in save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, verbose)
   1275         _write_raw(fname, self, info, picks, fmt, data_type, reset_range,
   1276                    start, stop, buffer_size, projector, inv_comp,
-> 1277                    drop_small_buffer, split_size, 0, None)
   1278 
   1279     def plot(self, events=None, duration=10.0, start=0.0, n_channels=20,

/Users/romain/Downloads/src/mne/mne/io/base.py in _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start, stop, buffer_size, projector, inv_comp, drop_small_buffer, split_size, part_idx, prev_fname)
   2066 
   2067         if picks is None:
-> 2068             data, times = raw[:, first:last]
   2069         else:
   2070             data, times = raw[picks, first:last]

/Users/romain/Downloads/src/mne/mne/io/base.py in **getitem**(self, item)
    546             data = self._read_segment(start=start, stop=stop, sel=sel,
    547                                       projector=self._projector,
--> 548                                       verbose=self.verbose)
    549         times = self.times[start:stop]
    550         return data, times

/Users/romain/Downloads/src/mne/mne/io/base.py in _read_segment(self, start, stop, sel, data_buffer, projector, verbose)
    383             self._read_segment_file(data[:, this_sl], idx, fi,
    384                                     int(start_file), int(stop_file),
--> 385                                     cals, mult)
    386             offset += n_read
    387 

/Users/romain/Downloads/src/mne/mne/io/ctf/ctf.py in _read_segment_file(self, data, idx, fi, start, stop, cals, mult)

/Users/romain/Downloads/src/mne/mne/utils.py in verbose(function, _args, *_kwargs)
    549         finally:
    550             set_log_level(old_level)
--> 551     return function(_args, *_kwargs)
    552 
    553 

/Users/romain/Downloads/src/mne/mne/io/ctf/ctf.py in _read_segment_file(self, data, idx, fi, start, stop, cals, mult)
    129                 this_data = np.fromstring(
    130                     fid.read(si['n_chan'] \* n_read \* 4), '>i4')
--> 131                 this_data.shape = (si['n_chan'], n_read)
    132                 this_data = this_data[:, r_lims[bi, 0]:r_lims[bi, 1]]
    133                 data_view = data[:, d_lims[bi, 0]:d_lims[bi, 1]]

## ValueError: total size of new array must be unchanged

Thanks a lot for your feedback. I can send you a link toward my ctf folder if you want to test.

I suspect your data file is truncated. Can you check? If you think it's not truncated, feel free to share a link and I'll take a look.

Actually, even if it is truncated, please share a link. I'd like to add a saner error message.

I am uploading the folder in dropbox. I will send you the link when it is done (slow internet connection).
I guess it is truncated, we checked the head position several time during the block. What I can do if this is the case? Thank you!

If it is indeed truncated, I can make it so it outputs as much good data as there is, and warns that there is less than expected. That's probably better than throwing an error. My guess is that's what Matti's C code has already done, though. I'll have to take a look to see.

Sorry for the time, it was not possible with my connection yesterday... Here is a link for the ctf data: https://www.dropbox.com/s/j1z6xuvfqyhwjhh/rom_wmtask_20151203_01.ds.zip?dl=0

Great, I'll see what I can figure out

It looks like your system clock channel goes to zero early on in the recording, indicating that the recording has ended (?). At least that's how the conversion routine treats the data. Does that make sense?

In other words, the system clock channel is nonzero for the first 158400 samples, then goes to zero, so we take that to mean the rest of the recording (737730 samples) is invalid.

We could easily add a `ignore_clock` option to the mne-python reader. But I don't know if it's the right thing to do or not. @mshamalainen any insight on how or why the `CTF` system clock channel could drop to zero after 158400 samples, leaving 737730 samples from the data file unused?


Resolving patch:
diff --git a/mne/io/ctf/ctf.py b/mne/io/ctf/ctf.py
--- a/mne/io/ctf/ctf.py
+++ b/mne/io/ctf/ctf.py
@@ -23,13 +23,19 @@
 from .constants import CTF
 
 
-def read_raw_ctf(directory, preload=False, verbose=None):
+def read_raw_ctf(directory, system_clock='truncate', preload=False,
+                 verbose=None):
     """Raw object from CTF directory
 
     Parameters
     ----------
     directory : str
         Path to the KIT data (ending in ``'.ds'``).
+    system_clock : str
+        How to treat the system clock. Use "truncate" (default) to truncate
+        the data file when the system clock drops to zero, and use "ignore"
+        to ignore the system clock (e.g., if head positions are measured
+        multiple times during a recording).
     preload : bool or str (default False)
         Preload data into memory for data manipulation and faster indexing.
         If True, the data will be preloaded into memory (fast, requires
@@ -47,8 +53,12 @@ def read_raw_ctf(directory, preload=False, verbose=None):
     See Also
     --------
     mne.io.Raw : Documentation of attribute and methods.
+
+    Notes
+    -----
+    .. versionadded:: 0.11
     """
-    return RawCTF(directory, preload, verbose)
+    return RawCTF(directory, system_clock, preload=preload, verbose=verbose)
 
 
 class RawCTF(_BaseRaw):
@@ -58,6 +68,11 @@ class RawCTF(_BaseRaw):
     ----------
     directory : str
         Path to the KIT data (ending in ``'.ds'``).
+    system_clock : str
+        How to treat the system clock. Use "truncate" (default) to truncate
+        the data file when the system clock drops to zero, and use "ignore"
+        to ignore the system clock (e.g., if head positions are measured
+        multiple times during a recording).
     preload : bool or str (default False)
         Preload data into memory for data manipulation and faster indexing.
         If True, the data will be preloaded into memory (fast, requires
@@ -72,13 +87,19 @@ class RawCTF(_BaseRaw):
     mne.io.Raw : Documentation of attribute and methods.
     """
     @verbose
-    def __init__(self, directory, preload=False, verbose=None):
+    def __init__(self, directory, system_clock='truncate', preload=False,
+                 verbose=None):
         # adapted from mne_ctf2fiff.c
         if not isinstance(directory, string_types) or \
                 not directory.endswith('.ds'):
             raise TypeError('directory must be a directory ending with ".ds"')
         if not op.isdir(directory):
             raise ValueError('directory does not exist: "%s"' % directory)
+        known_types = ['ignore', 'truncate']
+        if not isinstance(system_clock, string_types) or \
+                system_clock not in known_types:
+            raise ValueError('system_clock must be one of %s, not %s'
+                             % (known_types, system_clock))
         logger.info('ds directory : %s' % directory)
         res4 = _read_res4(directory)  # Read the magical res4 file
         coils = _read_hc(directory)  # Read the coil locations
@@ -97,7 +118,7 @@ def __init__(self, directory, preload=False, verbose=None):
             if meg4_name is None:
                 break
             # check how much data is in the file
-            sample_info = _get_sample_info(meg4_name, res4)
+            sample_info = _get_sample_info(meg4_name, res4, system_clock)
             if sample_info['n_samp'] == 0:
                 break
             if len(fnames) == 0:
@@ -135,7 +156,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
                 offset += n_read
 
 
-def _get_sample_info(fname, res4):
+def _get_sample_info(fname, res4, system_clock):
     """Helper to determine the number of valid samples"""
     logger.info('Finding samples for %s: ' % (fname,))
     if CTF.SYSTEM_CLOCK_CH in res4['ch_names']:
@@ -160,13 +181,15 @@ def _get_sample_info(fname, res4):
         if clock_ch is None:
             logger.info('    System clock channel is not available, assuming '
                         'all samples to be valid.')
-        else:
+        elif system_clock == 'ignore':
+            logger.info('    System clock channel is available, but ignored.')
+        else:  # use it
             logger.info('    System clock channel is available, checking '
                         'which samples are valid.')
             for t in range(n_trial):
                 # Skip to the correct trial
-                offset = CTF.HEADER_SIZE + (t * (res4['nsamp'] *
-                                                 res4['nchan']) +
+                samp_offset = t * res4['nsamp']
+                offset = CTF.HEADER_SIZE + (samp_offset * res4['nchan'] +
                                             (clock_ch * res4['nsamp'])) * 4
                 fid.seek(offset, 0)
                 this_data = np.fromstring(fid.read(4 * res4['nsamp']), '>i4')
@@ -175,7 +198,7 @@ def _get_sample_info(fname, res4):
                                        % (t + 1))
                 end = np.where(this_data == 0)[0]
                 if len(end) > 0:
-                    n_samp = offset + end[0]
+                    n_samp = samp_offset + end[0]
                     break
     if n_samp < res4['nsamp']:
         n_trial = 1
@@ -183,7 +206,7 @@ def _get_sample_info(fname, res4):
                     % (n_trial, n_samp, n_samp, res4['nchan']))
     else:
         n_trial = n_samp // res4['nsamp']
-        n_omit = n_samp % res4['nsamp']
+        n_omit = n_samp_tot - n_samp
         n_samp = n_trial * res4['nsamp']
         logger.info('    %d x %d = %d samples from %d chs'
                     % (n_trial, res4['nsamp'], n_samp, res4['nchan']))


Test patch:
diff --git a/mne/io/ctf/tests/test_ctf.py b/mne/io/ctf/tests/test_ctf.py
--- a/mne/io/ctf/tests/test_ctf.py
+++ b/mne/io/ctf/tests/test_ctf.py
@@ -163,5 +163,9 @@ def test_read_ctf():
         assert_allclose(raw[:][0], raw_c[:][0])
     assert_raises(TypeError, read_raw_ctf, 1)
     assert_raises(ValueError, read_raw_ctf, ctf_fname_continuous + 'foo.ds')
+    # test ignoring of system clock
+    read_raw_ctf(op.join(ctf_dir, ctf_fname_continuous), 'ignore')
+    assert_raises(ValueError, read_raw_ctf,
+                  op.join(ctf_dir, ctf_fname_continuous), 'foo')
 
 run_tests_if_main()