Repository: mne-tools/mne-bids

Issue:
write_raw_bids changes the data dtype on write for fif files
if you convert a fif files the data get systematically written to disk in float32 (default format of raw.save).

this is bad. The data buffers should not be changed.

@jasmainak thoughts?



Discussion:
I saw the pb when using old fils stored in short and the written files were twice bigger...
agree that the dtype should not change
Wow, good catch

Resolving patch:
diff --git a/mne_bids/write.py b/mne_bids/write.py
--- a/mne_bids/write.py
+++ b/mne_bids/write.py
@@ -662,7 +662,8 @@ def _write_raw_fif(raw, bids_fname):
         should be saved.
 
     """
-    raw.save(bids_fname, split_naming='bids', overwrite=True)
+    raw.save(bids_fname, fmt=raw.orig_format, split_naming='bids',
+             overwrite=True)
 
 
 def _write_raw_brainvision(raw, bids_fname, events):
@@ -679,8 +680,8 @@ def _write_raw_brainvision(raw, bids_fname, events):
         The events as MNE-Python format ndaray.
 
     """
-    if not check_version('pybv', '0.2'):
-        raise ImportError('pybv >=0.2.0 is required for converting '
+    if not check_version('pybv', '0.3'):
+        raise ImportError('pybv >=0.3 is required for converting '
                           'file to Brainvision format')
     from pybv import write_brainvision
     # Subtract raw.first_samp because brainvision marks events starting from
@@ -691,12 +692,27 @@ def _write_raw_brainvision(raw, bids_fname, events):
     meas_date = raw.info['meas_date']
     if meas_date is not None:
         meas_date = _stamp_to_dt(meas_date)
-    write_brainvision(data=raw.get_data(), sfreq=raw.info['sfreq'],
+
+    # XXX Always write as float32 to avoid precision loss during I/O roundtrip.
+    # See https://github.com/bids-standard/pybv/issues/45
+    fmt = 'binary_float32'
+    if raw.orig_format != 'single':
+        warn(f'Encountered data in "{raw.orig_format}" format. '
+             f'Converting to float32.', RuntimeWarning)
+
+    # writing with a resolution 1e-9 should be precise enough to avoid loss of
+    # resolution for any kind of data.
+    resolution = 1e-9
+    unit = 'µV'  # for compatibility with BrainVision software
+    write_brainvision(data=raw.get_data(),
+                      sfreq=raw.info['sfreq'],
                       ch_names=raw.ch_names,
                       fname_base=op.splitext(op.basename(bids_fname))[0],
                       folder_out=op.dirname(bids_fname),
                       events=events,
-                      resolution=1e-9,
+                      resolution=resolution,
+                      unit=unit,
+                      fmt=fmt,
                       meas_date=meas_date)
 
 
@@ -1087,7 +1103,8 @@ def write_raw_bids(raw, bids_path, events_data=None,
             bids_path.update(extension='.fif')
         elif bids_path.datatype in ['eeg', 'ieeg'] and ext != '.vhdr':
             if verbose:
-                warn('Converting to BV for anonymization')
+                warn('Converting data files to BrainVision format '
+                     'for anonymization')
             convert = True
             bids_path.update(extension='.vhdr')
 


Test patch:
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -23,6 +23,7 @@ jobs:
 
     env:
       TZ: Europe/Berlin
+      FORCE_COLOR: true
     steps:
     - uses: actions/checkout@v2
     - name: Set up Node.js
@@ -57,10 +58,10 @@ jobs:
     - name: Install BIDS validator (master)
       if: "matrix.bids-validator == 'master'"
       run: |
-        cd ..
+        pushd ..
         git clone --depth 1 https://github.com/bids-standard/bids-validator
         npm install -g bids-validator/bids-validator
-        cd mne-bids
+        popd
 
     - name: Display versions and environment information
       run: |
@@ -79,7 +80,7 @@ jobs:
         export BIDS_VALIDATOR_VERSION=`bids-validator --version`
         echo Using bids-validator $BIDS_VALIDATOR_VERSION
         python -m pytest . --cov=mne_bids mne_bids/tests/ mne_bids/commands/tests/ --cov-report=xml --cov-config=setup.cfg --verbose --ignore mne-python
-      shell: bash  
+      shell: bash
     - name: Run style & documentation tests
       if: "matrix.os == 'ubuntu-latest'"
       run: make pep
diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py
--- a/mne_bids/tests/test_write.py
+++ b/mne_bids/tests/test_write.py
@@ -271,7 +271,7 @@ def test_create_fif(_bids_validate):
     _bids_validate(bids_root)
 
 
-@requires_version('pybv', '0.2.0')
+@requires_version('pybv', '0.3')
 @pytest.mark.filterwarnings(warning_str['channel_unit_changed'])
 def test_fif(_bids_validate):
     """Test functionality of the write_raw_bids conversion for fif."""
@@ -543,6 +543,28 @@ def test_fif(_bids_validate):
         write_raw_bids(raw, bids_path)
 
 
+@pytest.mark.filterwarnings(warning_str['channel_unit_changed'])
+def test_fif_dtype(_bids_validate):
+    """Test functionality of the write_raw_bids conversion for fif."""
+    bids_root = _TempDir()
+    bids_path = _bids_path.copy().update(root=bids_root, datatype='meg')
+    data_path = testing.data_path()
+    raw_fname = op.join(data_path, 'MEG', 'sample',
+                        'sample_audvis_trunc_raw.fif')
+    desired_fmt = 'int'
+    raw = _read_raw_fif(raw_fname)
+
+    # Fiddle with raw.orig_format -- this should never be done in "real-life",
+    # but we do it here to test whether write_raw_bids() will actually stick
+    # to the format that's specified in that attribute.
+    assert raw.orig_format != desired_fmt  # We're actually changing something
+    raw.orig_format = desired_fmt
+
+    write_raw_bids(raw, bids_path, overwrite=False)
+    raw = read_raw_bids(bids_path)
+    assert raw.orig_format == desired_fmt
+
+
 def test_fif_anonymize(_bids_validate):
     """Test write_raw_bids() with anonymization fif."""
     bids_root = _TempDir()
@@ -845,7 +867,7 @@ def test_vhdr(_bids_validate):
     assert len([f for f in os.listdir(data_path) if op.isfile(f)]) == 0
 
     # test anonymize and convert
-    if check_version('pybv', '0.2.0'):
+    if check_version('pybv', '0.3'):
         raw = _read_raw_brainvision(raw_fname)
         output_path = _test_anonymize(raw, bids_path)
         _bids_validate(output_path)
@@ -904,7 +926,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
 
     raw = reader(raw_fname)
     events, events_id = mne.events_from_annotations(raw, event_id=None)
-    bids_output_path = write_raw_bids(raw, bids_path, overwrite=True)
+    kwargs = dict(raw=raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        bids_output_path = write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            bids_output_path = write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            bids_output_path = write_raw_bids(**kwargs)
+
     event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3,
                 'Visual/Right': 4, 'Smiley': 5, 'Button': 32}
 
@@ -936,7 +969,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
     raw.set_channel_types({'EMG': 'emg'})
 
     # Test we can overwrite dataset_description.json
-    write_raw_bids(raw, bids_path, overwrite=True)
+    kwargs = dict(raw=raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            write_raw_bids(**kwargs)
+
     make_dataset_description(bids_root, name="test",
                              authors=["test1", "test2"], overwrite=True)
     dataset_description_fpath = op.join(bids_root, "dataset_description.json")
@@ -946,7 +990,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
 
     # After writing the entire dataset again, dataset_description.json should
     # contain the default values.
-    write_raw_bids(raw, bids_path, overwrite=True)
+    kwargs = dict(raw=raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            write_raw_bids(**kwargs)
+
     with open(dataset_description_fpath, 'r') as f:
         dataset_description_json = json.load(f)
         assert dataset_description_json["Authors"] == \
@@ -995,7 +1050,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
                                                 lpa=[0, 1, 0],
                                                 rpa=[0, 0, 1])
     raw.set_montage(eeg_montage)
-    write_raw_bids(raw, bids_fname, overwrite=True)
+    kwargs = dict(raw=raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            write_raw_bids(**kwargs)
+
     electrodes_fpath = _find_matching_sidecar(bids_fname,
                                               suffix='electrodes',
                                               extension='.tsv')
@@ -1019,12 +1085,24 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
     assert len(list(data.values())[0]) == 2
 
     # check that scans list is properly converted to brainvision
-    if check_version('pybv', '0.2.0'):
+    if check_version('pybv', '0.3'):
         daysback_min, daysback_max = _get_anonymization_daysback(raw)
         daysback = (daysback_min + daysback_max) // 2
-        write_raw_bids(raw, bids_path,
-                       anonymize=dict(daysback=daysback),
-                       overwrite=True)
+        kwargs = dict(raw=raw, bids_path=bids_path,
+                      anonymize=dict(daysback=daysback), overwrite=True)
+        if dir_name == 'EDF':
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "int" format'):
+                write_raw_bids(**kwargs)
+        elif dir_name == 'Persyst':
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "double" format'):
+                write_raw_bids(**kwargs)
+        else:
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "short" format'):
+                write_raw_bids(**kwargs)
+
         data = _from_tsv(scans_tsv)
         bids_fname = bids_path.copy().update(suffix='eeg',
                                              extension='.vhdr')
@@ -1044,7 +1122,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
                                 for i in eeg_picks})
     bids_root = _TempDir()
     bids_path.update(root=bids_root)
-    write_raw_bids(ieeg_raw, bids_path)
+    kwargs = dict(raw=ieeg_raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            write_raw_bids(**kwargs)
+
     _bids_validate(bids_root)
 
     # assert README has references in it
@@ -1065,7 +1154,18 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
     ieeg_raw.set_montage(ecog_montage)
     bids_root = _TempDir()
     bids_path.update(root=bids_root)
-    write_raw_bids(ieeg_raw, bids_path)
+    kwargs = dict(raw=ieeg_raw, bids_path=bids_path, overwrite=True)
+    if dir_name == 'EDF':
+        write_raw_bids(**kwargs)
+    elif dir_name == 'NihonKohden':
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "short" format'):
+            write_raw_bids(**kwargs)
+    else:
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            write_raw_bids(**kwargs)
+
     _bids_validate(bids_root)
 
     # XXX: Should be improved with additional coordinate system descriptions
@@ -1084,9 +1184,24 @@ def test_eegieeg(dir_name, fname, reader, _bids_validate):
     assert coordsystem_json['iEEGCoordinateSystem'] == 'Other'
 
     # test anonymize and convert
-    if check_version('pybv', '0.2.0'):
+    if check_version('pybv', '0.3'):
         raw = reader(raw_fname)
-        output_path = _test_anonymize(raw, bids_path)
+        kwargs = dict(raw=raw, bids_path=bids_path, overwrite=True)
+        if dir_name == 'NihonKohden':
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "short" format'):
+                write_raw_bids(**kwargs)
+                output_path = _test_anonymize(raw, bids_path)
+        elif dir_name == 'EDF':
+            write_raw_bids(**kwargs)  # Just copies.
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "int" format'):
+                output_path = _test_anonymize(raw, bids_path)
+        else:
+            with pytest.warns(RuntimeWarning,
+                              match='Encountered data in "double" format'):
+                write_raw_bids(**kwargs)  # Converts.
+                output_path = _test_anonymize(raw, bids_path)
         _bids_validate(output_path)
 
 
@@ -1142,9 +1257,11 @@ def test_bdf(_bids_validate):
         write_raw_bids(raw, bids_path)
 
     # test anonymize and convert
-    if check_version('pybv', '0.2.0'):
+    if check_version('pybv', '0.3'):
         raw = _read_raw_bdf(raw_fname)
-        output_path = _test_anonymize(raw, bids_path)
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "int" format'):
+            output_path = _test_anonymize(raw, bids_path)
         _bids_validate(output_path)
 
 
@@ -1195,8 +1312,10 @@ def test_set(_bids_validate):
     _bids_validate(bids_root)
 
     # test anonymize and convert
-    if check_version('pybv', '0.2.0'):
-        output_path = _test_anonymize(raw, bids_path)
+    if check_version('pybv', '0.3'):
+        with pytest.warns(RuntimeWarning,
+                          match='Encountered data in "double" format'):
+            output_path = _test_anonymize(raw, bids_path)
         _bids_validate(output_path)
 
 
@@ -1726,13 +1845,14 @@ def test_mark_bad_channels_files():
     # TODO: allow write_brain_vision to write different units
     # mark bad channels that get stored as uV in write_brain_vision
     bads = ['CP5', 'CP6', 'HL', 'HR', 'Vb', 'ReRef']
-    mark_bad_channels(bads, bids_path=bids_path, overwrite=False)
+
+    with pytest.warns(RuntimeWarning,
+                      match='Encountered data in "short" format'):
+        mark_bad_channels(bads, bids_path=bids_path, overwrite=False)
     raw.info['bads'].extend(bads)
 
-    # the raw data should match without the bads
+    # the raw data should match if you drop the bads
     raw_2 = read_raw_bids(bids_path)
-
-    # if you drop the bads they should match
     raw.drop_channels(raw.info['bads'])
     raw_2.drop_channels(raw_2.info['bads'])
     assert_array_almost_equal(raw.get_data(), raw_2.get_data())