Repository: mne-tools/mne-python

Issue:
Error reading big CNT file
MNE Group:
I tried to read several cnt files of different sizes using `mne.io.read_raw_cnt`.
```py
data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None, stim_channel=True)
```
If the cnt file is lower than 2G, the file can be successfully loaded.
However, if the file size exceeds 2G, I will receive the following error:
```sh
<ipython-input-25-acf0568a07e0>:1: RuntimeWarning: Could not define the number of bytes automatically. Defaulting to 2.
  data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None, stim_channel=True)
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-25-acf0568a07e0> in <module>
----> 1 data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None, stim_channel=True)

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in read_raw_cnt(input_fname, montage, eog, misc, ecg, emg, data_format, date_format, preload, stim_channel, verbose)
    176     return RawCNT(input_fname, montage=montage, eog=eog, misc=misc, ecg=ecg,
    177                   emg=emg, data_format=data_format, date_format=date_format,
--> 178                   preload=preload, stim_channel=stim_channel, verbose=verbose)
    179
    180

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in __init__(self, input_fname, montage, eog, misc, ecg, emg, data_format, date_format, preload, stim_channel, verbose)
    430         input_fname = path.abspath(input_fname)
    431         info, cnt_info = _get_cnt_info(input_fname, eog, ecg, emg, misc,
--> 432                                        data_format, _date_format, stim_channel)
    433         last_samps = [cnt_info['n_samples'] - 1]
    434         _check_update_montage(info, montage)

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in _get_cnt_info(input_fname, eog, ecg, emg, misc, data_format, date_format, stim_channel_toggle)
    293         if stim_channel_toggle:
    294             data_format = 'int32' if n_bytes == 4 else 'int16'
--> 295             annot = _read_annotations_cnt(input_fname, data_format=data_format)
    296             events = (np.stack((annot.onset * sfreq,
    297                                 annot.duration * sfreq,

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in _read_annotations_cnt(fname, data_format)
     68
     69     with open(fname, 'rb') as fid:
---> 70         teeg = _read_teeg(fid, teeg_offset=event_table_pos)
     71
     72     event_parser = _get_event_parser(event_type=teeg.event_type)

~/.local/lib/python3.5/site-packages/mne/io/cnt/_utils.py in _read_teeg(f, teeg_offset)
     26     teeg_parser = Struct('<Bll')
     27
---> 28     f.seek(teeg_offset)
     29     return Teeg(*teeg_parser.unpack(f.read(teeg_parser.size)))
     30

OSError: [Errno 22] Invalid argument
```

Also:
If I use:
```py
data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None)
```
I got different error:
```sh
<ipython-input-30-dbbb56936797>:1: FutureWarning: The parameter `stim_channel` controlling the stim channel synthesis has not been specified. In 0.18 it defaults to True but will change to False in 0.19 (when no stim channel synthesis will be allowed) and be removed in 0.20; migrate code to use `stim_channel=False` and :func:`mne.events_from_annotations` or set `stim_channel=True` to avoid this warning.
  data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None)
<ipython-input-30-dbbb56936797>:1: RuntimeWarning: Could not define the number of bytes automatically. Defaulting to 2.
  data = mne.io.read_raw_cnt('test.cnt', preload=True, montage=None)
Reading 0 ... 4528779  =      0.000 ...  4528.779 secs...
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
<ipython-input-30-dbbb56936797> in <module>
----> 1 data = mne.io.read_raw_cnt(test.cnt', preload=True, montage=None)

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in read_raw_cnt(input_fname, montage, eog, misc, ecg, emg, data_format, date_format, preload, stim_channel, verbose)
    176     return RawCNT(input_fname, montage=montage, eog=eog, misc=misc, ecg=ecg,
    177                   emg=emg, data_format=data_format, date_format=date_format,
--> 178                   preload=preload, stim_channel=stim_channel, verbose=verbose)
    179
    180

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in __init__(self, input_fname, montage, eog, misc, ecg, emg, data_format, date_format, preload, stim_channel, verbose)
    439         data_format = 'int32' if cnt_info['n_bytes'] == 4 else 'int16'
    440         self.set_annotations(
--> 441             _read_annotations_cnt(input_fname, data_format=data_format))
    442
    443     @verbose

~/.local/lib/python3.5/site-packages/mne/io/cnt/cnt.py in _read_annotations_cnt(fname, data_format)
     68
     69     with open(fname, 'rb') as fid:
---> 70         teeg = _read_teeg(fid, teeg_offset=event_table_pos)
     71
     72     event_parser = _get_event_parser(event_type=teeg.event_type)

~/.local/lib/python3.5/site-packages/mne/io/cnt/_utils.py in _read_teeg(f, teeg_offset)
     26     teeg_parser = Struct('<Bll')
     27
---> 28     f.seek(teeg_offset)
     29     return Teeg(*teeg_parser.unpack(f.read(teeg_parser.size)))
     30

OSError: [Errno 22] Invalid argument
```
Here is my mne info
```sh
mne.sys_info()
Platform:      Linux-4.4.0-151-generic-x86_64-with-Ubuntu-16.04-xenial
Python:        3.5.2 (default, Nov 12 2018, 13:43:14)  [GCC 5.4.0 20160609]
Executable:    /usr/bin/python3
CPU:           x86_64: 12 cores
Memory:        Unavailable (requires "psutil" package)
mne:           0.18.1
numpy:         1.16.3 {blas=openblas, lapack=openblas}
scipy:         1.3.0
matplotlib:    3.0.1 {backend=TkAgg}

sklearn:       0.19.1
nibabel:       Not found
mayavi:        Not found
cupy:          Not found
pandas:        0.22.0
dipy:          Not found
```



Discussion:
Since the CNT file is huge, which means the samples section in the CNT file (ref this: [Structure of CNT file ](http://paulbourke.net/dataformats/eeg/)) is also huge.

And mne choose to read the **event table position** from the SETUP section, which read as a long (32 bit) integer. 
 
``` python
with open(fname, 'rb') as fid:
        ....
        fid.seek(SETUP_EVENTTABLEPOS_OFFSET)
        (event_table_pos,) = np.frombuffer(fid.read(4), dtype='<i4')`
```
As a result, the maximum length of the CNT file mne supports is equals to : 
``` python
maximum_seconds = (np.iinfo(np.int32).max - 900 - 75 * n_channels) / sfreq / n_channels / 4
``` 
"900" is the size of the SETUP section, 75*n_channels is the size of ELECTLOC, and "4" is the bytes each sample point takes.

For the situation n_channels=62 and sfreq=1000,  the result is 8659 seconds (about 2.4 hours). When the length of your CNT file is too long that the value of EVENT_TABLE_POS will overflow. 

To solve this problem, you need to change the way to obtain the value of EVENT_TABLE_POS: calculate it using follow formula instead of read it from SETUP section:
``` python
--        # fid.seek(SETUP_EVENTTABLEPOS_OFFSET)         # Line 66
--        # (event_table_pos,) = np.frombuffer(fid.read(4), dtype='<i4')  # Line 67

++        fid.seek(864)
++        (n_samples,) = np.frombuffer(fid.read(4), dtype='<i4')
++        event_table_pos = 900 + 75 * int(n_channels) + 4 * int(n_channels) * int(n_samples)

--          event_offset = np.fromfile(fid, dtype='<i4', count=1)[0]     # Line 242
++        event_offset = 900 + 75 * int(n_channels) + 4 * int(n_channels) * int(n_samples)
```
any chance you can open a PR @AimerLee97 ? thx

>

Thank you all. It works!
However, the author said in his code that '# Header has a field for number of samples, but it does not seem to be too reliable. That's why we have option for setting n_bytes manually.' So I want to know if this update is stable. Because you use the nsamples in header to calculate the correct event_offset.
@AimerLee nice catch. I guess I broke it in #6047. I'll open a PR.
can anyone share (privately or publicly) one of those long CNT files ??
When trying these large files, the code I'm reading in `Teeg` field of `TEEG` is not 1 or 2 is a random number (usually 129 or 116) our codebase allows for 3 as well. Is there any extension that I'm not aware of with different structures or am I doing something wrong?

```sh
typedef struct {
   char Teeg;                   /* Either 1 or 2                           */
   long Size;                   /* Total length of all the events          */
   long Offset;                 /* Hopefully always 0                      */
} TEEG;
```


Resolving patch:
diff --git a/mne/io/cnt/_utils.py b/mne/io/cnt/_utils.py
--- a/mne/io/cnt/_utils.py
+++ b/mne/io/cnt/_utils.py
@@ -6,6 +6,9 @@
 from collections import namedtuple
 from math import modf
 from datetime import datetime
+from os import SEEK_END
+import numpy as np
+
 
 from ...utils import warn
 
@@ -93,3 +96,47 @@ def _session_date_2_meas_date(session_date, date_format):
         return None
     else:
         return (int_part, frac_part)
+
+
+def _compute_robust_event_table_position(fid, data_format='int32'):
+    """Compute `event_table_position`.
+
+    When recording event_table_position is computed (as accomulation). If the
+    file recording is large then this value overflows and ends up pointing
+    somewhere else. (SEE #gh-6535)
+
+    If the file is smaller than 2G the value in the SETUP is returned.
+    Otherwise, the address of the table position is computed from:
+    n_samples, n_channels, and the bytes size.
+    """
+    SETUP_NCHANNELS_OFFSET = 370
+    SETUP_NSAMPLES_OFFSET = 864
+    SETUP_EVENTTABLEPOS_OFFSET = 886
+
+    fid_origin = fid.tell()  # save the state
+
+    if fid.seek(0, SEEK_END) < 2e9:
+
+        fid.seek(SETUP_EVENTTABLEPOS_OFFSET)
+        (event_table_pos,) = np.frombuffer(fid.read(4), dtype='<i4')
+
+    else:
+        if data_format == 'auto':
+            warn('Using `data_format=\'auto\' for a CNT file larger'
+                 ' than 2Gb is not granted to work. Please pass'
+                 ' \'int16\' or \'int32\'.` (assuming int32)')
+
+        n_bytes = 2 if data_format == 'int16' else 4
+
+        fid.seek(SETUP_NSAMPLES_OFFSET)
+        (n_samples,) = np.frombuffer(fid.read(4), dtype='<i4')
+
+        fid.seek(SETUP_NCHANNELS_OFFSET)
+        (n_channels,) = np.frombuffer(fid.read(2), dtype='<u2')
+
+        event_table_pos = (900 +
+                           75 * int(n_channels) +
+                           n_bytes * int(n_channels) * int(n_samples))
+
+    fid.seek(fid_origin)  # restore the state
+    return event_table_pos
diff --git a/mne/io/cnt/cnt.py b/mne/io/cnt/cnt.py
--- a/mne/io/cnt/cnt.py
+++ b/mne/io/cnt/cnt.py
@@ -19,7 +19,7 @@
 
 
 from ._utils import (_read_teeg, _get_event_parser, _session_date_2_meas_date,
-                     CNTEventType3)
+                     _compute_robust_event_table_position, CNTEventType3)
 
 
 def _read_annotations_cnt(fname, data_format='int16'):
@@ -45,7 +45,6 @@ def _read_annotations_cnt(fname, data_format='int16'):
     # Offsets from SETUP structure in http://paulbourke.net/dataformats/eeg/
     SETUP_NCHANNELS_OFFSET = 370
     SETUP_RATE_OFFSET = 376
-    SETUP_EVENTTABLEPOS_OFFSET = 886
 
     def _translating_function(offset, n_channels, event_type,
                               data_format=data_format):
@@ -63,8 +62,8 @@ def _translating_function(offset, n_channels, event_type,
         fid.seek(SETUP_RATE_OFFSET)
         (sfreq,) = np.frombuffer(fid.read(2), dtype='<u2')
 
-        fid.seek(SETUP_EVENTTABLEPOS_OFFSET)
-        (event_table_pos,) = np.frombuffer(fid.read(4), dtype='<i4')
+        event_table_pos = _compute_robust_event_table_position(
+            fid=fid, data_format=data_format)
 
     with open(fname, 'rb') as fid:
         teeg = _read_teeg(fid, teeg_offset=event_table_pos)
@@ -233,8 +232,10 @@ def _get_cnt_info(input_fname, eog, ecg, emg, misc, data_format, date_format,
         fid.seek(2, 1)
         highcutoff = np.fromfile(fid, dtype='f4', count=1)[0]
 
-        fid.seek(886)
-        event_offset = np.fromfile(fid, dtype='<i4', count=1)[0]
+        event_offset = _compute_robust_event_table_position(
+            fid=fid, data_format=data_format
+        )
+        fid.seek(890)
         cnt_info['continuous_seconds'] = np.fromfile(fid, dtype='<f4',
                                                      count=1)[0]
 
@@ -256,14 +257,18 @@ def _get_cnt_info(input_fname, eog, ecg, emg, misc, data_format, date_format,
         else:
             n_bytes = 2 if data_format == 'int16' else 4
             n_samples = data_size // (n_bytes * n_channels)
+
         # Channel offset refers to the size of blocks per channel in the file.
         cnt_info['channel_offset'] = np.fromfile(fid, dtype='<i4', count=1)[0]
         if cnt_info['channel_offset'] > 1:
             cnt_info['channel_offset'] //= n_bytes
         else:
             cnt_info['channel_offset'] = 1
-        ch_names, cals, baselines, chs, pos = (list(), list(), list(), list(),
-                                               list())
+
+        ch_names, cals, baselines, chs, pos = (
+            list(), list(), list(), list(), list()
+        )
+
         bads = list()
         for ch_idx in range(n_channels):  # ELECTLOC fields
             fid.seek(data_offset + 75 * ch_idx)


Test patch: