Repository: mne-tools/mne-bids

Issue:
[BUG] invalid info added to raw.info upon reading
Hi, 
mne-bids seems to have read out my participants.tsv and added info to the `subject_info` field of the `raw.info`. Later when I read/write the raw data again, I get an error related to invalid values in the info:
```
ValueError                Traceback (most recent call last)
Input In [4], in <cell line: 1>()
----> 1 raw_proc.save(deriv_stub % 'filt-raw_sss.fif', overwrite=True, fmt='single', split_naming='bids')` `File <decorator-gen-213>:12, in save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, split_naming, verbose)` `File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:1489, in BaseRaw.save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, split_naming, verbose)
  1487 _validate_type(split_naming, str, 'split_naming')
  1488 _check_option('split_naming', split_naming, ('neuromag', 'bids'))
-> 1489 _write_raw(fname, self, info, picks, fmt, data_type, reset_range,
  1490      start, stop, buffer_size, projector, drop_small_buffer,
  1491      split_size, split_naming, 0, None, overwrite)
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:2224, in _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start, stop, buffer_size, projector, drop_small_buffer, split_size, split_naming, part_idx, prev_fname, overwrite)
  2222 picks = _picks_to_idx(info, picks, 'all', ())
  2223 with start_and_end_file(use_fname) as fid:
-> 2224   cals = _start_writing_raw(fid, info, picks, data_type,
  2225                reset_range, raw.annotations)
  2226   with ctx:
  2227     final_fname = _write_raw_fid(
  2228       raw, info, picks, fid, cals, part_idx, start, stop,
  2229       buffer_size, prev_fname, split_size, use_fname,
  (...)
  2232       overwrite=True # we've started writing already above
  2233     )
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:2427, in _start_writing_raw(fid, info, sel, data_type, reset_range, annotations)
  2424     info['chs'][k]['range'] = 1.0
  2425   cals.append(info['chs'][k]['cal'] * info['chs'][k]['range'])
-> 2427 write_meas_info(fid, info, data_type=data_type, reset_range=reset_range)
  2429 #
  2430 # Annotations
  2431 #
  2432 if len(annotations) > 0: # don't save empty annot
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/meas_info.py:2146, in write_meas_info(fid, info, data_type, reset_range)
  2144   write_int(fid, FIFF.FIFF_SUBJ_HAND, si['hand'])
  2145 if si.get('weight') is not None:
-> 2146   write_float(fid, FIFF.FIFF_SUBJ_WEIGHT, si['weight'])
  2147 if si.get('height') is not None:
  2148   write_float(fid, FIFF.FIFF_SUBJ_HEIGHT, si['height'])
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/write.py:96, in write_float(fid, kind, data)
   94 """Write a single-precision floating point tag to a fif file."""
   95 data_size = 4
---> 96 data = np.array(data, dtype='>f4').T
   97 _write(fid, data, kind, data_size, FIFF.FIFFT_FLOAT, '>f4')
ValueError: could not convert string to float: 'n/a'ValueError                Traceback (most recent call last)
Input In [4], in <cell line: 1>()
----> 1 raw_proc.save(deriv_stub % 'filt-raw_sss.fif', overwrite=True, fmt='single', split_naming='bids')` `File <decorator-gen-213>:12, in save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, split_naming, verbose)` `File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:1489, in BaseRaw.save(self, fname, picks, tmin, tmax, buffer_size_sec, drop_small_buffer, proj, fmt, overwrite, split_size, split_naming, verbose)
  1487 _validate_type(split_naming, str, 'split_naming')
  1488 _check_option('split_naming', split_naming, ('neuromag', 'bids'))
-> 1489 _write_raw(fname, self, info, picks, fmt, data_type, reset_range,
  1490      start, stop, buffer_size, projector, drop_small_buffer,
  1491      split_size, split_naming, 0, None, overwrite)
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:2224, in _write_raw(fname, raw, info, picks, fmt, data_type, reset_range, start, stop, buffer_size, projector, drop_small_buffer, split_size, split_naming, part_idx, prev_fname, overwrite)
  2222 picks = _picks_to_idx(info, picks, 'all', ())
  2223 with start_and_end_file(use_fname) as fid:
-> 2224   cals = _start_writing_raw(fid, info, picks, data_type,
  2225                reset_range, raw.annotations)
  2226   with ctx:
  2227     final_fname = _write_raw_fid(
  2228       raw, info, picks, fid, cals, part_idx, start, stop,
  2229       buffer_size, prev_fname, split_size, use_fname,
  (...)
  2232       overwrite=True # we've started writing already above
  2233     )
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/base.py:2427, in _start_writing_raw(fid, info, sel, data_type, reset_range, annotations)
  2424     info['chs'][k]['range'] = 1.0
  2425   cals.append(info['chs'][k]['cal'] * info['chs'][k]['range'])
-> 2427 write_meas_info(fid, info, data_type=data_type, reset_range=reset_range)
  2429 #
  2430 # Annotations
  2431 #
  2432 if len(annotations) > 0: # don't save empty annot
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/meas_info.py:2146, in write_meas_info(fid, info, data_type, reset_range)
  2144   write_int(fid, FIFF.FIFF_SUBJ_HAND, si['hand'])
  2145 if si.get('weight') is not None:
-> 2146   write_float(fid, FIFF.FIFF_SUBJ_WEIGHT, si['weight'])
  2147 if si.get('height') is not None:
  2148   write_float(fid, FIFF.FIFF_SUBJ_HEIGHT, si['height'])
File /gpfs/project/projects/bpsydm/tools/pyEnvs/meg/lib/python3.9/site-packages/mne/io/write.py:96, in write_float(fid, kind, data)
   94 """Write a single-precision floating point tag to a fif file."""
   95 data_size = 4
---> 96 data = np.array(data, dtype='>f4').T
   97 _write(fid, data, kind, data_size, FIFF.FIFFT_FLOAT, '>f4')
ValueError: could not convert string to float: 'n/a'
```

See [here](https://mne.discourse.group/t/strange-artifact-with-neuromag-meg-data/5331/6) for details. Next week, I will provide more details about the specific situation in which this happens.




Discussion:

Resolving patch:
diff --git a/mne_bids/read.py b/mne_bids/read.py
--- a/mne_bids/read.py
+++ b/mne_bids/read.py
@@ -194,10 +194,11 @@ def _handle_participants_reading(participants_fname, raw, subject):
     participants_tsv = _from_tsv(participants_fname)
     subjects = participants_tsv['participant_id']
     row_ind = subjects.index(subject)
+    raw.info['subject_info'] = dict()  # start from scratch
 
     # set data from participants tsv into subject_info
     for col_name, value in participants_tsv.items():
-        if col_name == 'sex' or col_name == 'hand':
+        if col_name in ('sex', 'hand'):
             value = _map_options(what=col_name, key=value[row_ind],
                                  fro='bids', to='mne')
             # We don't know how to translate to MNE, so skip.
@@ -206,15 +207,24 @@ def _handle_participants_reading(participants_fname, raw, subject):
                     info_str = 'subject sex'
                 else:
                     info_str = 'subject handedness'
-                warn(f'Unable to map `{col_name}` value to MNE. '
+                warn(f'Unable to map "{col_name}" value "{value}" to MNE. '
                      f'Not setting {info_str}.')
+        elif col_name in ('height', 'weight'):
+            try:
+                value = float(value[row_ind])
+            except ValueError:
+                value = None
         else:
-            value = value[row_ind]
+            if value[row_ind] == 'n/a':
+                value = None
+            else:
+                value = value[row_ind]
+
         # add data into raw.Info
-        if raw.info['subject_info'] is None:
-            raw.info['subject_info'] = dict()
         key = 'his_id' if col_name == 'participant_id' else col_name
-        raw.info['subject_info'][key] = value
+        if value is not None:
+            assert key not in raw.info['subject_info']
+            raw.info['subject_info'][key] = value
 
     return raw
 
@@ -763,6 +773,7 @@ def read_raw_bids(bids_path, extra_params=None, verbose=None):
         )
     else:
         warn(f"participants.tsv file not found for {raw_path}")
+        raw.info['subject_info'] = dict()
 
     assert raw.annotations.orig_time == raw.info['meas_date']
     return raw
diff --git a/mne_bids/write.py b/mne_bids/write.py
--- a/mne_bids/write.py
+++ b/mne_bids/write.py
@@ -341,10 +341,13 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False):
         False, an error will be raised.
 
     """
-    subject_age = "n/a"
-    sex = "n/a"
+    subject_age = 'n/a'
+    sex = 'n/a'
     hand = 'n/a'
+    weight = 'n/a'
+    height = 'n/a'
     subject_info = raw.info.get('subject_info', None)
+
     if subject_id != 'emptyroom' and subject_info is not None:
         # add sex
         sex = _map_options(what='sex', key=subject_info.get('sex', 0),
@@ -371,18 +374,34 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False):
         else:
             subject_age = "n/a"
 
+        # add weight and height
+        weight = subject_info.get('weight', 'n/a')
+        height = subject_info.get('height', 'n/a')
+
     subject_id = 'sub-' + subject_id
     data = OrderedDict(participant_id=[subject_id])
-    data.update({'age': [subject_age], 'sex': [sex], 'hand': [hand]})
+    data.update({
+        'age': [subject_age],
+        'sex': [sex],
+        'hand': [hand],
+        'weight': [weight],
+        'height': [height]
+    })
 
     if os.path.exists(fname):
         orig_data = _from_tsv(fname)
         # whether the new data exists identically in the previous data
-        exact_included = _contains_row(orig_data,
-                                       {'participant_id': subject_id,
-                                        'age': subject_age,
-                                        'sex': sex,
-                                        'hand': hand})
+        exact_included = _contains_row(
+            data=orig_data,
+            row_data={
+                'participant_id': subject_id,
+                'age': subject_age,
+                'sex': sex,
+                'hand': hand,
+                'weight': weight,
+                'height': height
+            }
+        )
         # whether the subject id is in the previous data
         sid_included = subject_id in orig_data['participant_id']
         # if the subject data provided is different to the currently existing
@@ -440,26 +459,53 @@ def _participants_json(fname, overwrite=False):
         an error will be raised.
 
     """
-    cols = OrderedDict()
-    cols['participant_id'] = {'Description': 'Unique participant identifier'}
-    cols['age'] = {'Description': 'Age of the participant at time of testing',
-                   'Units': 'years'}
-    cols['sex'] = {'Description': 'Biological sex of the participant',
-                   'Levels': {'F': 'female', 'M': 'male'}}
-    cols['hand'] = {'Description': 'Handedness of the participant',
-                    'Levels': {'R': 'right', 'L': 'left', 'A': 'ambidextrous'}}
+    data = {
+        'participant_id': {
+            'Description': 'Unique participant identifier'
+        },
+        'age': {
+            'Description': 'Age of the participant at time of testing',
+            'Units': 'years'
+        },
+        'sex': {
+            'Description': 'Biological sex of the participant',
+            'Levels': {
+                'F': 'female',
+                'M': 'male'
+            }
+        },
+        'hand': {
+            'Description': 'Handedness of the participant',
+            'Levels': {
+                'R': 'right',
+                'L': 'left',
+                'A': 'ambidextrous'
+            }
+        },
+        'weight': {
+            'Description': 'Body weight of the participant',
+            'Units': 'kg'
+        },
+        'height': {
+            'Description': 'Body height of the participant',
+            'Units': 'm'
+        }
+    }
 
     # make sure to append any JSON fields added by the user
     # Note: mne-bids will overwrite age, sex and hand fields
     # if `overwrite` is True
-    if op.exists(fname):
-        with open(fname, 'r', encoding='utf-8-sig') as fin:
-            orig_cols = json.load(fin, object_pairs_hook=OrderedDict)
+    fname = Path(fname)
+    if fname.exists():
+        orig_cols = json.loads(
+            fname.read_text(encoding='utf-8'),
+            object_pairs_hook=OrderedDict
+        )
         for key, val in orig_cols.items():
-            if key not in cols:
-                cols[key] = val
+            if key not in data:
+                data[key] = val
 
-    _write_json(fname, cols, overwrite)
+    _write_json(fname, data, overwrite)
 
 
 def _scans_tsv(raw, raw_fname, fname, keep_source, overwrite=False):


Test patch:
diff --git a/mne_bids/tests/test_read.py b/mne_bids/tests/test_read.py
--- a/mne_bids/tests/test_read.py
+++ b/mne_bids/tests/test_read.py
@@ -120,13 +120,15 @@ def test_read_participants_data(tmp_path):
     subject_info = {
         'hand': 1,
         'sex': 2,
+        'weight': 70.5,
+        'height': 180.5
     }
     raw.info['subject_info'] = subject_info
     write_raw_bids(raw, bids_path, overwrite=True, verbose=False)
     raw = read_raw_bids(bids_path=bids_path)
-    print(raw.info['subject_info'])
     assert raw.info['subject_info']['hand'] == 1
-    assert raw.info['subject_info']['sex'] == 2
+    assert raw.info['subject_info']['weight'] == 70.5
+    assert raw.info['subject_info']['height'] == 180.5
     assert raw.info['subject_info'].get('birthday', None) is None
     assert raw.info['subject_info']['his_id'] == f'sub-{bids_path.subject}'
     assert 'participant_id' not in raw.info['subject_info']
@@ -139,25 +141,35 @@ def test_read_participants_data(tmp_path):
     raw = read_raw_bids(bids_path=bids_path)
     assert raw.info['subject_info']['hand'] == 0
     assert raw.info['subject_info']['sex'] == 2
+    assert raw.info['subject_info']['weight'] == 70.5
+    assert raw.info['subject_info']['height'] == 180.5
     assert raw.info['subject_info'].get('birthday', None) is None
 
     # make sure things are read even if the entries don't make sense
     participants_tsv = _from_tsv(participants_tsv_fpath)
     participants_tsv['hand'][0] = 'righty'
     participants_tsv['sex'][0] = 'malesy'
+    # 'n/a' values should get omitted
+    participants_tsv['weight'] = ['n/a']
+    participants_tsv['height'] = ['tall']
+
     _to_tsv(participants_tsv, participants_tsv_fpath)
     with pytest.warns(RuntimeWarning, match='Unable to map'):
         raw = read_raw_bids(bids_path=bids_path)
-        assert raw.info['subject_info']['hand'] is None
-        assert raw.info['subject_info']['sex'] is None
 
-    # make sure to read in if no participants file
+    assert 'hand' not in raw.info['subject_info']
+    assert 'sex' not in raw.info['subject_info']
+    assert 'weight' not in raw.info['subject_info']
+    assert 'height' not in raw.info['subject_info']
+
+    # test reading if participants.tsv is missing
     raw = _read_raw_fif(raw_fname, verbose=False)
     write_raw_bids(raw, bids_path, overwrite=True, verbose=False)
-    os.remove(participants_tsv_fpath)
+    participants_tsv_fpath.unlink()
     with pytest.warns(RuntimeWarning, match='participants.tsv file not found'):
         raw = read_raw_bids(bids_path=bids_path)
-        assert raw.info['subject_info'] is None
+
+    assert raw.info['subject_info'] == dict()
 
 
 @pytest.mark.parametrize(