Repository: mne-tools/mne-python

Issue:
Improve mne anomymize 
mne.anonymize should be doing similar things in cpp and python

cc @LorenzE


Discussion:
@juangpc and @imsorryk are almost done working on a mne-cpp version of a fiff anonymization tool. PR can be found [here](https://github.com/mne-tools/mne-cpp/pull/393). I propose we try to finish the cpp version first and then port functionality to the existing mne-pyhton version. What do you guys think? 
@bloyl expressed interest in working on this. I'm fine with waiting or if @bloyl wants to get started based on the concepts from the C code hopefully the final product wouldn't require too much adjustment
@bloyl  and I will start to compare the current anonymized files from both mne-python and mne-cpp. I attached some slides ([mne_anonymize.pdf](https://github.com/mne-tools/mne-python/files/3676937/mne_anonymize.pdf)) by @juangpc and @imsorryk about the new mne_anonymize application. We are currently implementing solution 2. Tags which we are currently scanning for can be found in the switch case [here](https://github.com/mne-tools/mne-cpp/blob/dcc3875a15be93577515a96dc37befaf7133769b/applications/mne_anonymize/fiffanonymizer.cpp#L536). 


@juangpc https://github.com/juangpc/fiff_anonymizer/blob/master/fiff_anonymizer.m should me moved to mne-matlab repo https://github.com/mne-tools/mne-matlab thanks for open a PR.

Resolving patch:
diff --git a/mne/channels/channels.py b/mne/channels/channels.py
--- a/mne/channels/channels.py
+++ b/mne/channels/channels.py
@@ -573,17 +573,14 @@ def plot_sensors(self, kind='topomap', ch_type=None, title=None,
                             show=show)
 
     @copy_function_doc_to_method_doc(anonymize_info)
-    def anonymize(self):
+    def anonymize(self, daysback=None, keep_his=False):
         """
         .. versionadded:: 0.13.0
         """
-        anonymize_info(self.info)
+        anonymize_info(self.info, daysback=daysback, keep_his=keep_his)
         if hasattr(self, 'annotations'):
-            # XXX : anonymize should rather subtract a random date
-            # rather than setting it to None
-            self.annotations.orig_time = None
+            self.annotations.orig_time = self.info['meas_date']
             self.annotations.onset -= self._first_time
-
         return self
 
 
diff --git a/mne/commands/mne_anonymize.py b/mne/commands/mne_anonymize.py
new file mode 100644
--- /dev/null
+++ b/mne/commands/mne_anonymize.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# Authors : Dominik Krzeminski
+#           Luke Bloy <luke.bloy@gmail.com>
+
+"""Anonymize raw fif file.
+
+To anonymize other file types call :func:`mne.io.anonymize_info` on their
+`info` objects and resave to disk.
+
+Examples
+--------
+.. code-block:: console
+
+    $ mne anonymize -f sample_audvis_raw.fif
+
+"""
+
+import sys
+import mne
+import os.path as op
+
+ANONYMIZE_FILE_PREFIX = 'anon'
+
+
+def mne_anonymize(fif_fname, out_fname, keep_his, daysback, overwrite):
+    """Call *anonymize_info* on fif file and save.
+
+    Parameters
+    ----------
+    fif_fname : str
+        Raw fif File
+    out_fname : str | None
+        Output file name
+        relative paths are saved relative to parent dir of fif_fname
+        None will save to parent dir of fif_fname with default prefix
+    daysback : int | None
+        Number of days to subtract from all dates.
+        If None will default to move date of service to Jan 1 2000
+    keep_his : bool
+        If True his_id of subject_info will NOT be overwritten.
+        defaults to False
+    overwrite : bool
+        Overwrite output file if it already exists
+    """
+    raw = mne.io.read_raw_fif(fif_fname, allow_maxshield=True)
+    raw.anonymize(daysback=daysback, keep_his=keep_his)
+
+    # determine out_fname
+    dir_name = op.split(fif_fname)[0]
+    if out_fname is None:
+        fif_bname = op.basename(fif_fname)
+        out_fname = op.join(dir_name,
+                            "{}-{}".format(ANONYMIZE_FILE_PREFIX, fif_bname))
+    elif not op.isabs(out_fname):
+        out_fname = op.join(dir_name, out_fname)
+
+    raw.save(out_fname, overwrite=overwrite)
+
+
+def run():
+    """Run *mne_anonymize* command."""
+    from mne.commands.utils import get_optparser
+
+    parser = get_optparser(__file__)
+
+    parser.add_option("-f", "--file", type="string", dest="file",
+                      help="Name of file to modify.", metavar="FILE",
+                      default=None)
+    parser.add_option("-o", "--output", type="string", dest="output",
+                      help="Name of anonymized output file."
+                      "`anon-` prefix is added to FILE if not given",
+                      metavar="OUTFILE", default=None)
+    parser.add_option("--keep_his", dest="keep_his", action="store_true",
+                      help="Keep the HIS tag (not advised)", default=False)
+    parser.add_option("-d", "--daysback", type="int", dest="daysback",
+                      help="Move dates in file backwards by this many days.",
+                      metavar="N_DAYS", default=None)
+    parser.add_option("--overwrite", dest="overwrite", action="store_true",
+                      help="Overwrite input file.", default=False)
+
+    options, args = parser.parse_args()
+    if options.file is None:
+        parser.print_help()
+        sys.exit(1)
+
+    fname = options.file
+    out_fname = options.output
+    keep_his = options.keep_his
+    daysback = options.daysback
+    overwrite = options.overwrite
+    if not fname.endswith('.fif'):
+        raise ValueError('%s does not seem to be a .fif file.' % fname)
+
+    mne_anonymize(fname, out_fname, keep_his, daysback, overwrite)
+
+
+is_main = (__name__ == '__main__')
+if is_main:
+    run()
diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py
--- a/mne/io/meas_info.py
+++ b/mne/io/meas_info.py
@@ -119,13 +119,19 @@ def _summarize_str(st):
     return st[:56][::-1].split(',', 1)[-1][::-1] + ', ...'
 
 
-def _stamp_to_dt(stamp):
+def _dt_to_stamp(inp_date):
+    """Convert a datetime object to a meas_date."""
+    return int(inp_date.timestamp() // 1), inp_date.microsecond
+
+
+def _stamp_to_dt(utc_stamp):
     """Convert timestamp to datetime object in Windows-friendly way."""
     # The min on windows is 86400
-    stamp = [int(s) for s in stamp]
+    stamp = [int(s) for s in utc_stamp]
     if len(stamp) == 1:  # In case there is no microseconds information
         stamp.append(0)
-    return (datetime.datetime.utcfromtimestamp(stamp[0]) +
+    return (datetime.datetime.fromtimestamp(stamp[0],
+                                            tz=datetime.timezone.utc) +
             datetime.timedelta(0, 0, stamp[1]))  # day, sec, μs
 
 
@@ -1861,16 +1867,19 @@ def _force_update_info(info_base, info_target):
             i_targ[key] = val
 
 
-def anonymize_info(info):
+def anonymize_info(info, daysback=None, keep_his=False):
     """Anonymize measurement information in place.
 
-    Reset 'subject_info', 'meas_date', 'file_id', and 'meas_id' keys if they
-    exist in ``info``.
-
     Parameters
     ----------
     info : dict, instance of Info
         Measurement information for the dataset.
+    daysback : int | None
+        Number of days to subtract from all dates.
+        If None (default) the date of service will be set to Jan 1ˢᵗ 2000.
+    keep_his : bool
+        If True his_id of subject_info will NOT be overwritten.
+        Defaults to False.
 
     Returns
     -------
@@ -1879,18 +1888,114 @@ def anonymize_info(info):
 
     Notes
     -----
+    Removes potentially identifying information if it exist in ``info``.
+    Specifically for each of the following we use:
+
+    - meas_date, file_id, meas_id
+          A default value, or as specified by ``daysback``.
+    - subject_info
+          Default values, except for 'birthday' which is adjusted
+          to maintain the subject age.
+    - experimenter, proj_name, description
+          Default strings.
+    - utc_offset
+          ``None``.
+    - proj_id
+          Zeros.
+    - proc_history
+          Dates use the meas_date logic, and experimenter a default string.
+    - helium_info, device_info
+          Dates use the meas_date logic, meta info uses defaults.
+
     Operates in place.
     """
     _validate_type(info, 'info', "self")
-    if info.get('subject_info') is not None:
-        del info['subject_info']
-    info['meas_date'] = None
+
+    default_anon_dos = datetime.datetime(2000, 1, 1, 0, 0, 0,
+                                         tzinfo=datetime.timezone.utc)
+    default_str = "mne_anonymize"
+    default_subject_id = 0
+    default_desc = ("Anonymized using a time shift"
+                    " to preserve age at acquisition")
+
+    # datetime object representing meas_date
+    meas_date_datetime = _stamp_to_dt(info['meas_date'])
+
+    if daysback is None:
+        delta_t = meas_date_datetime - default_anon_dos
+    else:
+        delta_t = datetime.timedelta(days=daysback)
+
+    # adjust meas_date
+    info['meas_date'] = _dt_to_stamp(meas_date_datetime - delta_t)
+
+    # file_id and meas_id
     for key in ('file_id', 'meas_id'):
         value = info.get(key)
         if value is not None:
             assert 'msecs' not in value
-            value['secs'] = DATE_NONE[0]
-            value['usecs'] = DATE_NONE[1]
+            value['secs'] = info['meas_date'][0]
+            value['usecs'] = info['meas_date'][1]
+            value['machid'][:] = 0
+
+    # subject info
+    subject_info = info.get('subject_info')
+    if subject_info is not None:
+        if subject_info.get('id') is not None:
+            subject_info['id'] = default_subject_id
+        if keep_his:
+            logger.warning('Not fully anonymizing info - keeping \'his_id\'')
+        elif subject_info.get('his_id') is not None:
+            subject_info['his_id'] = str(default_subject_id)
+
+        for key in ('last_name', 'first_name', 'middle_name'):
+            if subject_info.get(key) is not None:
+                subject_info[key] = default_str
+
+        if subject_info.get('birthday') is not None:
+            dob = datetime.datetime(subject_info['birthday'][0],
+                                    subject_info['birthday'][1],
+                                    subject_info['birthday'][2])
+            dob -= delta_t
+            subject_info['birthday'] = dob.year, dob.month, dob.day
+
+        for key in ('weight', 'height'):
+            if subject_info.get(key) is not None:
+                subject_info[key] = 0
+
+    info['experimenter'] = default_str
+    info['description'] = default_desc
+
+    if info['proj_id'] is not None:
+        info['proj_id'][:] = 0
+    if info['proj_name'] is not None:
+        info['proj_name'] = default_str
+    if info['utc_offset'] is not None:
+        info['utc_offset'] = None
+
+    proc_hist = info.get('proc_history')
+    if proc_hist is not None:
+        for record in proc_hist:
+            record['block_id']['secs'] = info['meas_date'][0]
+            record['block_id']['usecs'] = info['meas_date'][1]
+            record['block_id']['machid'][:] = 0
+            record['date'] = info['meas_date']
+            record['experimenter'] = default_str
+
+    hi = info.get('helium_info')
+    if hi is not None:
+        if hi.get('orig_file_guid') is not None:
+            hi['orig_file_guid'] = default_str
+        if hi.get('meas_date') is not None:
+            hi['meas_date'] = [info['meas_date'][0],
+                               info['meas_date'][1]]
+
+    di = info.get('device_info')
+    if di is not None:
+        for k in ('serial', 'site'):
+            if di.get(k) is not None:
+                di[k] = default_str
+
     return info
 
 


Test patch:
diff --git a/mne/commands/tests/test_commands.py b/mne/commands/tests/test_commands.py
--- a/mne/commands/tests/test_commands.py
+++ b/mne/commands/tests/test_commands.py
@@ -15,9 +15,10 @@
                           mne_make_scalp_surfaces, mne_maxfilter,
                           mne_report, mne_surf2bem, mne_watershed_bem,
                           mne_compare_fiff, mne_flash_bem, mne_show_fiff,
-                          mne_show_info, mne_what, mne_setup_source_space)
+                          mne_show_info, mne_what, mne_setup_source_space,
+                          mne_anonymize)
 from mne.datasets import testing, sample
-from mne.io import read_raw_fif
+from mne.io import read_raw_fif, read_info
 from mne.utils import (run_tests_if_main, requires_mne,
                        requires_mayavi, requires_tvtk, requires_freesurfer,
                        traits_test, ArgvSetter, modified_env)
@@ -314,4 +315,14 @@ def test_show_info():
         mne_show_info.run()
 
 
+def test_anonymize(tmpdir):
+    """Test mne anonymize."""
+    check_usage(mne_anonymize)
+    out_fname = op.join(tmpdir, 'anon_test_raw.fif')
+    with ArgvSetter(('-f', raw_fname, '-o', out_fname)):
+        mne_anonymize.run()
+    info = read_info(out_fname)
+    assert(op.exists(out_fname))
+    assert_equal(info['meas_date'], (946684800, 0))
+
 run_tests_if_main()
diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py
--- a/mne/io/tests/test_meas_info.py
+++ b/mne/io/tests/test_meas_info.py
@@ -6,7 +6,7 @@
 
 import hashlib
 import os.path as op
-
+from datetime import datetime, timedelta, timezone
 
 import pytest
 import numpy as np
@@ -17,13 +17,13 @@
 from mne.event import make_fixed_length_events
 from mne.datasets import testing
 from mne.io import (read_fiducials, write_fiducials, _coil_trans_to_loc,
-                    _loc_to_coil_trans, read_raw_fif, read_info, write_info,
-                    anonymize_info)
+                    _loc_to_coil_trans, read_raw_fif, read_info, write_info)
 from mne.io.constants import FIFF
-from mne.io.write import DATE_NONE, _generate_meas_id
+from mne.io.write import _generate_meas_id
 from mne.io.meas_info import (Info, create_info, _merge_info,
                               _force_update_info, RAW_INFO_FIELDS,
-                              _bad_chans_comp, _get_valid_units)
+                              _bad_chans_comp, _get_valid_units,
+                              anonymize_info, _stamp_to_dt, _dt_to_stamp)
 from mne.io._digitization import (_write_dig_points, _read_dig_points,
                                   _make_dig_points,)
 from mne.io import read_raw_ctf
@@ -427,46 +427,89 @@ def test_check_consistency():
     assert_array_equal(info3['ch_names'], ['a', 'b-0', 'b-1', 'c', 'b-2'])
 
 
-def _is_anonymous(inst):
-    """Check all the anonymity fields.
-
-    inst is either a raw or epochs object.
-    """
-    from collections import namedtuple
-    anonymity_checks = namedtuple("anonymity_checks",
-                                  ["missing_subject_info",
-                                   "anonymous_file_id_secs",
-                                   "anonymous_file_id_usecs",
-                                   "anonymous_meas_id_secs",
-                                   "anonymous_meas_id_usecs",
-                                   "anonymous_meas_date",
-                                   "anonymous_annotations"])
-
-    if 'subject_info' not in inst.info.keys():
-        missing_subject_info = True
-    else:
-        missing_subject_info = inst.info['subject_info'] is None
-
-    anonymous_file_id_secs = inst.info['file_id']['secs'] == DATE_NONE[0]
-    anonymous_file_id_usecs = inst.info['file_id']['usecs'] == DATE_NONE[1]
-    anonymous_meas_id_secs = inst.info['meas_id']['secs'] == DATE_NONE[0]
-    anonymous_meas_id_usecs = inst.info['meas_id']['usecs'] == DATE_NONE[1]
-    if inst.info['meas_date'] is None:
-        anonymous_meas_date = True
-    else:
-        assert isinstance(inst.info['meas_date'], tuple)
-        anonymous_meas_date = inst.info['meas_date'] == DATE_NONE
-
-    anonymous_annotations = (hasattr(inst, 'annotations') and
-                             inst.annotations.orig_time is None)
-
-    return anonymity_checks(missing_subject_info,
-                            anonymous_file_id_secs,
-                            anonymous_file_id_usecs,
-                            anonymous_meas_id_secs,
-                            anonymous_meas_id_usecs,
-                            anonymous_meas_date,
-                            anonymous_annotations)
+def _test_anonymize_info(base_info):
+    """Test that sensitive information can be anonymized."""
+    pytest.raises(TypeError, anonymize_info, 'foo')
+
+    default_anon_dos = datetime(2000, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+    default_str = "mne_anonymize"
+    default_subject_id = 0
+    default_desc = ("Anonymized using a time shift" +
+                    " to preserve age at acquisition")
+
+    # Test no error for incomplete info
+    info = base_info.copy()
+    info.pop('file_id')
+    anonymize_info(info)
+
+    # Fake some subject data
+    meas_date = datetime(2010, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+    base_info['meas_date'] = _dt_to_stamp(meas_date)
+
+    base_info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar',
+                                     first_name='bar', birthday=(1987, 4, 8),
+                                     sex=0, hand=1)
+
+    # generate expected info...
+    # first expected result with no options.
+    # will move DOS from 2010/1/1 to 2000/1/1 which is 3653 days.
+    exp_info = base_info.copy()
+    exp_info['description'] = default_desc
+    exp_info['experimenter'] = default_str
+    exp_info['proj_name'] = default_str
+    exp_info['proj_id'][:] = 0
+    exp_info['subject_info']['first_name'] = default_str
+    exp_info['subject_info']['last_name'] = default_str
+    exp_info['subject_info']['id'] = default_subject_id
+    exp_info['subject_info']['his_id'] = str(default_subject_id)
+    # this bday is 3653 days different. the change in day is due to a
+    # different number of leap days between 1987 and 1977 than between
+    # 2010 and 2000.
+    exp_info['subject_info']['birthday'] = (1977, 4, 7)
+    exp_info['meas_date'] = _dt_to_stamp(default_anon_dos)
+    for key in ('file_id', 'meas_id'):
+        value = exp_info.get(key)
+        if value is not None:
+            assert 'msecs' not in value
+            value['secs'] = exp_info['meas_date'][0]
+            value['usecs'] = exp_info['meas_date'][1]
+            value['machid'][:] = 0
+
+    # exp 2 tests the keep_his option
+    exp_info_2 = exp_info.copy()
+    exp_info_2['subject_info']['his_id'] = 'foobar'
+
+    # exp 3 tests is a supplied daysback
+    dt = timedelta(days=43)
+    exp_info_3 = exp_info.copy()
+    exp_info_3['subject_info']['birthday'] = (1987, 2, 24)
+    exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt)
+    for key in ('file_id', 'meas_id'):
+        value = exp_info_3.get(key)
+        if value is not None:
+            assert 'msecs' not in value
+            value['secs'] = exp_info_3['meas_date'][0]
+            value['usecs'] = exp_info_3['meas_date'][1]
+            value['machid'][:] = 0
+
+    new_info = anonymize_info(base_info.copy())
+    assert_object_equal(new_info, exp_info)
+
+    new_info = anonymize_info(base_info.copy(), keep_his=True)
+    assert_object_equal(new_info, exp_info_2)
+
+    new_info = anonymize_info(base_info.copy(), daysback=dt.days)
+    assert_object_equal(new_info, exp_info_3)
+
+
+def test_meas_date_convert(tmpdir):
+    """Test conversions of meas_date to datetime objects."""
+    meas_date = (1346981585, 835782)
+    meas_datetime = _stamp_to_dt(meas_date)
+    meas_date2 = _dt_to_stamp(meas_datetime)
+    assert(meas_date == meas_date2)
+    assert(meas_datetime == datetime(2012, 9, 7, 1, 33, 5, 835782,
+                                     tzinfo=timezone.utc))
 
 
 def test_anonymize(tmpdir):
@@ -479,32 +522,17 @@ def test_anonymize(tmpdir):
                                     duration=[1, 1],
                                     description='dummy',
                                     orig_time=None))
-    raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar',
-                                    first_name='bar', birthday=(1987, 4, 8),
-                                    sex=0, hand=1)
-
-    # Test no error for incomplete info
-    info = raw.info.copy()
-    info.pop('file_id')
-    anonymize_info(info)
 
     # Test instance method
     events = read_events(event_name)
     epochs = Epochs(raw, events[:1], 2, 0., 0.1, baseline=None)
 
-    assert not any(_is_anonymous(raw))
-    raw.anonymize()
-    assert all(_is_anonymous(raw))
-
-    assert not any(_is_anonymous(epochs)[:-1])  # epochs has no annotations
-    epochs.anonymize()
-    assert all(_is_anonymous(epochs)[:-1])
+    _test_anonymize_info(raw.info.copy())
+    _test_anonymize_info(epochs.info.copy())
 
-    # When we write out with raw.save, these get overwritten with the
-    # new save time
-    out_fname = tmpdir.join('test_subj_info_raw.fif')
-    raw.save(out_fname, overwrite=True)
-    assert all(_is_anonymous(read_raw_fif(out_fname)))
+    # test that annotations are correctly zeroed
+    raw.anonymize()
+    assert(raw.annotations.orig_time is raw.info['meas_date'])
 
 
 @testing.requires_testing_data
diff --git a/mne/tests/test_report.py b/mne/tests/test_report.py
--- a/mne/tests/test_report.py
+++ b/mne/tests/test_report.py
@@ -22,6 +22,7 @@
 from mne.utils import (_TempDir, requires_mayavi, requires_nibabel, Bunch,
                        run_tests_if_main, traits_test, requires_h5py)
 from mne.viz import plot_alignment
+from mne.io.write import DATE_NONE
 
 data_dir = testing.data_path(download=False)
 subjects_dir = op.join(data_dir, 'subjects')
@@ -165,13 +166,29 @@ def test_report_raw_psd_and_date():
     assert 'PSD' in ''.join(report.html)
     assert 'GMT' in ''.join(report.html)
 
-    # DATE_NONE functionality
+    # test new anonymize functionality
     report = Report()
     raw.anonymize()
     raw.save(raw_fname_new, overwrite=True)
     report.parse_folder(data_path=tempdir, render_bem=False,
                         on_error='raise')
     assert isinstance(report.html, list)
+    assert 'GMT' in ''.join(report.html)
+
+    # DATE_NONE functionality
+    report = Report()
+    # old style (pre 0.20) date anonymization
+    raw.info['meas_date'] = None
+    for key in ('file_id', 'meas_id'):
+        value = raw.info.get(key)
+        if value is not None:
+            assert 'msecs' not in value
+            value['secs'] = DATE_NONE[0]
+            value['usecs'] = DATE_NONE[1]
+    raw.save(raw_fname_new, overwrite=True)
+    report.parse_folder(data_path=tempdir, render_bem=False,
+                        on_error='raise')
+    assert isinstance(report.html, list)
     assert 'GMT' not in ''.join(report.html)