Repository: mne-tools/mne-python

Issue:
save changes file_id, causes anonymize to fail
#### Describe the bug
Saving an instance of raw results in a file in which `info['file_id']['secs'] == 0`.
If you then load from this file and try to anonymize the result, an exception is raised because 'secs' becomes too small a number.


#### Steps to reproduce
```python
from pathlib import Path
import tempfile
from datetime import date

import mne

# Load
sample_dir = Path(mne.datasets.sample.data_path())
raw_path = sample_dir / 'MEG' / 'sample' / 'sample_audvis_raw.fif'
raw = mne.io.read_raw_fif(raw_path, verbose=False)
print('Original file_id\n', raw.info['file_id'])

# Save
temp_path = Path(tempfile.TemporaryDirectory().name) / 'sample_audvis_raw.fif'
temp_path.parent.mkdir()
raw.save(temp_path)

# Load
raw2 = mne.io.read_raw_fif(temp_path, verbose=False)
print('Updated file_id\n', raw2.info['file_id'])

# Anonymize
daysback = (raw2.info['meas_date'].date() - date(1924, 1, 1)).days
raw2.anonymize(daysback=daysback)
```


#### Expected results
No exceptions raised.


#### Actual results

```pytb
Original file_id
 {'version': 65537, 'machid': array([   1082367, 1060765696]), 'secs': 1038942072, 'usecs': 190548}
Updated file_id
 {'version': 65540, 'machid': array([808661043, 808661043]), 'secs': 0, 'usecs': 2147483647}
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-127-0f07c80ed58b> in <module>
     22 # Anonymize
     23 daysback = (raw2.info['meas_date'].date() - date(1924, 1, 1)).days
---> 24 raw2.anonymize(daysback=daysback)

<decorator-gen-35> in anonymize(self, daysback, keep_his, verbose)

E:\Evgenii_Kalenkovich\programs\miniminiconda3\envs\syntax_tagging\lib\site-packages\mne\channels\channels.py in anonymize(self, daysback, keep_his, verbose)
    617         """
    618         anonymize_info(self.info, daysback=daysback, keep_his=keep_his,
--> 619                        verbose=verbose)
    620         self.set_meas_date(self.info['meas_date'])  # unify annot update
    621         return self

<decorator-gen-30> in anonymize_info(info, daysback, keep_his, verbose)

E:\Evgenii_Kalenkovich\programs\miniminiconda3\envs\syntax_tagging\lib\site-packages\mne\io\meas_info.py in anonymize_info(info, daysback, keep_his, verbose)
   2264                 'daysback parameter was too large.'
   2265                 'Underlying Error:\n')
-> 2266     _check_dates(info, prepend_error=err_mesg)
   2267 
   2268     return info

E:\Evgenii_Kalenkovich\programs\miniminiconda3\envs\syntax_tagging\lib\site-packages\mne\io\meas_info.py in _check_dates(info, prepend_error)
   1478                                           np.iinfo('>i4').min,
   1479                                           np.iinfo('>i4').max,
-> 1480                                           value[key_2]),)
   1481 
   1482     meas_date = info.get('meas_date')

RuntimeError: anonymize_info generated an inconsistent info object. daysback parameter was too large.Underlying Error:
info[file_id][secs] must be between "-2147483648" and "2147483647", got "-2490564253"
```


#### Additional information

```yaml
Platform:      Windows-2012ServerR2-6.3.9600-SP0
Python:        3.7.4 (default, Aug  9 2019, 18:34:13) [MSC v.1915 64 bit (AMD64)]
Executable:    E:\Evgenii_Kalenkovich\programs\miniminiconda3\envs\syntax_tagging\python.exe
CPU:           Intel64 Family 6 Model 85 Stepping 7, GenuineIntel: 64 cores
Memory:        Unavailable (requires "psutil" package)
mne:           0.21.2
numpy:         1.16.3 {blas=mkl_rt, lapack=mkl_rt}
scipy:         1.2.1
matplotlib:    2.2.3 {backend=module://ipykernel.pylab.backend_inline}

sklearn:       0.21.3
numba:         Not found
nibabel:       3.2.1
cupy:          Not found
pandas:        0.25.1
dipy:          Not found
mayavi:        Not found
pyvista:       Not found
vtk:           Not found
```



Discussion:
can you share the file with me so I can look?

>

> can you share the file with me so I can look?
> […](#)

Sure... but the code above uses the sample dataset. I can, of course, send my file if this would help.
indeed thanks. it's enough

@larsoner you see immediately the pb?

>

> Updated file_id
>  {'version': 65540, 'machid': array([808661043, 808661043]), 'secs': 0, 'usecs': 2147483647}

This means the `file_id` is `DATE_NONE`, i.e., should be seen as `None` by whatever is processing it. I'm not sure why this would happen, though. It seems like a bug
Hi, @larsoner! Thank you for taking a look and confirming it is a bug. How would one work around it?
sorry about the delay. It turns out that the way we produce unique file_id is broken when combined with anonymization. We set it to 1970 (DATE_NONE) which is not really randomized ....

the MNE-C code does (in file_id.c):

```
int fiff_new_file_id (fiffId id)
     /*
      * Return a (hopefully) unique file id
      */
{
  int fixed_id[2];
  
  fixed_id[0] = fixed_id[1] = 0;
  if (fiff_get_machid(fixed_id) == FIFF_FAIL) {
    /*
     * Never mind...
       */
    fixed_id[0] = fixed_id[1] = 0;
  }
  /*
   * Internet address in the first two words
   */
  id->machid[0] = fixed_id[0];
  id->machid[1] = fixed_id[1];
  /*
   * Time in the third and fourth words
   */
  /*
   * Time in the third and fourth words
   * Since practically no system gives times in 
   * true micro seconds, the last three digits
   * are randomized to insure uniqueness.
   */
  {
    long secs,usecs;
    if (fiff_get_time(&secs, &usecs) == FIFF_FAIL)
      return FIFF_FAIL;
    id->time.secs  = secs;
    id->time.usecs = usecs + rand() % 1000;
  }
  id->version = FIFFC_VERSION;
  return FIFF_OK;
}
```

I see a problem in using a randomized approach as the file hash will change all the time...

@larsoner thoughts?

Resolving patch:
diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py
--- a/mne/io/meas_info.py
+++ b/mne/io/meas_info.py
@@ -2152,7 +2152,10 @@ def anonymize_info(info, daysback=None, keep_his=False, verbose=None):
         value = info.get(key)
         if value is not None:
             assert 'msecs' not in value
-            if none_meas_date:
+            if (none_meas_date or
+                    ((value['secs'], value['usecs']) == DATE_NONE)):
+                # Don't try to shift backwards in time when no measurement
+                # date is available or when file_id is already a place holder
                 tmp = DATE_NONE
             else:
                 tmp = _add_timedelta_to_stamp(
@@ -2242,7 +2245,7 @@ def anonymize_info(info, daysback=None, keep_his=False, verbose=None):
                 'Underlying Error:\n')
     info._check_consistency(prepend_error=err_mesg)
     err_mesg = ('anonymize_info generated an inconsistent info object. '
-                'daysback parameter was too large.'
+                'daysback parameter was too large. '
                 'Underlying Error:\n')
     _check_dates(info, prepend_error=err_mesg)
 


Test patch:
diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py
--- a/mne/io/tests/test_meas_info.py
+++ b/mne/io/tests/test_meas_info.py
@@ -6,7 +6,7 @@
 
 import hashlib
 import os.path as op
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta, timezone, date
 
 import pytest
 import numpy as np
@@ -667,6 +667,19 @@ def test_anonymize(tmpdir):
     assert_allclose(raw.annotations.onset, expected_onset)
 
 
+def test_anonymize_with_io(tmpdir):
+    """Test that IO does not break anonymization."""
+    raw = read_raw_fif(raw_fname)
+
+    temp_path = tmpdir.join('tmp_raw.fif')
+    raw.save(temp_path)
+
+    raw2 = read_raw_fif(temp_path)
+
+    daysback = (raw2.info['meas_date'].date() - date(1924, 1, 1)).days
+    raw2.anonymize(daysback=daysback)
+
+
 @testing.requires_testing_data
 def test_csr_csc(tmpdir):
     """Test CSR and CSC."""