Repository: mne-tools/mne-bids

Issue:
Participants JSON shouldn't overwrite existing components if added
#### Describe the bug
If you add `participants.json` fields that are specific to your data, and theen add a new dataset, it should not overwrite the participants.json file, but not overwrite the participants.tsv file.

#### Steps to reproduce
1. Modify participants.json and participants.tsv to add elements outside MNE-BIDs
2. write a new dataset, and participants.json will not retain the old fields, while participants.tsv does.

#### Expected results
They should both retain fields. 

#### Additional information
In addition, `overwrite=True/False` I don't think is sufficient. If you set overwrite to True, then wouldn't you just overwrite the whole participants files and then have to start from scratch adding these new fields in? 


Discussion:
the use case is not clear to me. If you add a new participant whose age and name doesn't match, then it's appended at the end. Otherwise, an error is thrown if overwrite = False. That would be the expected behavior
So I was thinking something along the lines of this:

```
# check to make sure participant data is overwritten, but keeps the fields
    participants_tsv = op.join(bids_root, 'participants.tsv')
    data['subject_outcome'] = ['n/a'] * len(data['participant_id'])
    data['subject_outcome'][data['participant_id'].index('sub-01')] = 'S'
    _to_tsv(data, participants_tsv)
    participants_json_fpath = op.join(bids_root, 'participants.json')
    _update_sidecar(participants_json_fpath, 'subject_outcome',
                    {'Description': 'trial-outcome',
                     'Levels': {'S': 'success', 'F': 'failure'}})
    write_raw_bids(raw, bids_basename, overwrite=True)
    data = _from_tsv(participants_tsv)
    assert data['age'][data['participant_id'].index('sub-01')] == '9'
    assert data['subject_outcome'][data['participant_id'].index('sub-01')] == 'S'
```
When you overwrite, I would presume, it should retain new fields added into the participants.json file, besides age, hand, and sex, which are the only ones supported in MNE-BIDs rn.
sorry for the late response.

I see, it seems like an exotic use case. However, I'm okay with it if you can do it without adding too much code complexity.

Resolving patch:
diff --git a/mne_bids/tsv_handler.py b/mne_bids/tsv_handler.py
--- a/mne_bids/tsv_handler.py
+++ b/mne_bids/tsv_handler.py
@@ -4,7 +4,7 @@
 from copy import deepcopy
 
 
-def _combine(data1, data2, drop_column=None):
+def _combine_rows(data1, data2, drop_column=None):
     """Add two OrderedDict's together and optionally drop repeated data.
 
     Parameters
@@ -22,7 +22,6 @@ def _combine(data1, data2, drop_column=None):
     -------
     data : collections.OrderedDict
         The new combined data.
-
     """
     data = deepcopy(data1)
     for key, value in data2.items():
diff --git a/mne_bids/write.py b/mne_bids/write.py
--- a/mne_bids/write.py
+++ b/mne_bids/write.py
@@ -7,6 +7,7 @@
 #          Matt Sanderson <matt.sanderson@mq.edu.au>
 #
 # License: BSD (3-clause)
+import json
 import os
 import os.path as op
 from datetime import datetime, date, timedelta, timezone
@@ -41,7 +42,8 @@
 from mne_bids.copyfiles import (copyfile_brainvision, copyfile_eeglab,
                                 copyfile_ctf, copyfile_bti, copyfile_kit)
 from mne_bids.read import reader
-from mne_bids.tsv_handler import _from_tsv, _combine, _drop, _contains_row
+from mne_bids.tsv_handler import (_from_tsv, _drop, _contains_row,
+                                  _combine_rows)
 
 from mne_bids.config import (ORIENTATION, UNITS, MANUFACTURERS,
                              IGNORED_CHANNELS, ALLOWED_EXTENSIONS,
@@ -277,8 +279,24 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False,
             raise FileExistsError('"%s" already exists in the participant '  # noqa: E501 F821
                                   'list. Please set overwrite to '
                                   'True.' % subject_id)
-        # otherwise add the new data
-        data = _combine(orig_data, data, 'participant_id')
+
+        # Append any additional columns that original data had.
+        # Keep the original order of the data by looping over
+        # the original OrderedDict keys
+        col_name = 'participant_id'
+        for key in orig_data.keys():
+            if key in data:
+                continue
+
+            # add original value for any user-appended columns
+            # that were not handled by mne-bids
+            p_id = data[col_name][0]
+            if p_id in orig_data[col_name]:
+                row_idx = orig_data[col_name].index(p_id)
+                data[key] = [orig_data[key][row_idx]]
+
+        # otherwise add the new data as new row
+        data = _combine_rows(orig_data, data, 'participant_id')
 
     # overwrite is forced to True as all issues with overwrite == False have
     # been handled by this point
@@ -310,6 +328,16 @@ def _participants_json(fname, overwrite=False, verbose=True):
     cols['hand'] = {'Description': 'Handedness of the participant',
                     'Levels': {'R': 'right', 'L': 'left', 'A': 'ambidextrous'}}
 
+    # make sure to append any JSON fields added by the user
+    # Note: mne-bids will overwrite age, sex and hand fields
+    # if `overwrite` is True
+    if op.exists(fname):
+        with open(fname, 'r') as fin:
+            orig_cols = json.load(fin, object_pairs_hook=OrderedDict)
+        for key, val in orig_cols.items():
+            if key not in cols:
+                cols[key] = val
+
     _write_json(fname, cols, overwrite, verbose)
 
 
@@ -354,7 +382,7 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False, verbose=True):
             raise FileExistsError('"%s" already exists in the scans list. '  # noqa: E501 F821
                                   'Please set overwrite to True.' % raw_fname)
         # otherwise add the new data
-        data = _combine(orig_data, data, 'filename')
+        data = _combine_rows(orig_data, data, 'filename')
 
     # overwrite is forced to True as all issues with overwrite == False have
     # been handled by this point
@@ -870,10 +898,15 @@ def write_raw_bids(raw, bids_basename, bids_root, events_data=None,
     overwrite : bool
         Whether to overwrite existing files or data in files.
         Defaults to False.
+
         If overwrite is True, any existing files with the same BIDS parameters
         will be overwritten with the exception of the `participants.tsv` and
         `scans.tsv` files. For these files, parts of pre-existing data that
-        match the current data will be replaced.
+        match the current data will be replaced. For `participants.tsv`,
+        specifically, age, sex and hand fields will be overwritten, while
+        any added fields in the `participants.json` and `participants.tsv`
+        by a user will be kept.
+
         If overwrite is False, no existing data will be overwritten or
         replaced.
     verbose : bool


Test patch:
diff --git a/mne_bids/tests/test_tsv_handler.py b/mne_bids/tests/test_tsv_handler.py
--- a/mne_bids/tests/test_tsv_handler.py
+++ b/mne_bids/tests/test_tsv_handler.py
@@ -15,7 +15,7 @@
                             category=ImportWarning)
     import mne  # noqa: F401
 
-from mne_bids.tsv_handler import (_from_tsv, _to_tsv, _combine, _drop,
+from mne_bids.tsv_handler import (_from_tsv, _to_tsv, _combine_rows, _drop,
                                   _contains_row, _tsv_to_str)
 import pytest
 
@@ -28,13 +28,13 @@ def test_tsv_handler():
     d = odict(a=[1, 2, 3, 4], b=['five', 'six', 'seven', 'eight'])
     assert _contains_row(d, {'a': 1, 'b': 'five'})
     d2 = odict(a=[5], b=['nine'])
-    d = _combine(d, d2)
+    d = _combine_rows(d, d2)
     assert 5 in d['a']
     d2 = odict(a=[5])
-    d = _combine(d, d2)
+    d = _combine_rows(d, d2)
     assert 'n/a' in d['b']
     d2 = odict(a=[5], b=['ten'])
-    d = _combine(d, d2, drop_column='a')
+    d = _combine_rows(d, d2, drop_column='a')
     # make sure that the repeated data was dropped
     assert 'nine' not in d['b']
     print(_tsv_to_str(d))
@@ -63,9 +63,9 @@ def test_tsv_handler():
     d2 = odict(a=[4], b=['five'], c=[3.1415])
     # raise error if a new column is tried to be added
     with pytest.raises(KeyError):
-        d = _combine(d, d2)
+        d = _combine_rows(d, d2)
     d2 = odict(a=[5])
-    d = _combine(d, d2)
+    d = _combine_rows(d, d2)
     assert d['b'] == ['three', 'four', 'n/a']
     assert _contains_row(d, {'a': 5})
 
diff --git a/mne_bids/tests/test_write.py b/mne_bids/tests/test_write.py
--- a/mne_bids/tests/test_write.py
+++ b/mne_bids/tests/test_write.py
@@ -44,7 +44,7 @@
 from mne_bids.write import (_stamp_to_dt, _get_anonymization_daysback,
                             get_anonymization_daysback)
 from mne_bids.tsv_handler import _from_tsv, _to_tsv
-from mne_bids.utils import _find_matching_sidecar
+from mne_bids.utils import _find_matching_sidecar, _update_sidecar
 from mne_bids.pick import coil_type
 
 base_path = op.join(op.dirname(mne.__file__), 'io')
@@ -273,6 +273,50 @@ def test_fif(_bids_validate):
     data = _from_tsv(participants_tsv)
     assert data['age'][data['participant_id'].index('sub-01')] == '9'
 
+    # check to make sure participant data is overwritten, but keeps the fields
+    data = _from_tsv(participants_tsv)
+    participant_idx = data['participant_id'].index(f'sub-{subject_id}')
+    # create a new test column in participants file tsv
+    data['subject_test_col1'] = ['n/a'] * len(data['participant_id'])
+    data['subject_test_col1'][participant_idx] = 'S'
+    data['test_col2'] = ['n/a'] * len(data['participant_id'])
+    orig_key_order = list(data.keys())
+    _to_tsv(data, participants_tsv)
+    # crate corresponding json entry
+    participants_json_fpath = op.join(bids_root, 'participants.json')
+    json_field = {
+        'Description': 'trial-outcome',
+        'Levels': {
+            'S': 'success',
+            'F': 'failure'
+        }
+    }
+    _update_sidecar(participants_json_fpath, 'subject_test_col1', json_field)
+    # bids root should still be valid because json reflects changes in tsv
+    _bids_validate(bids_root)
+    write_raw_bids(raw, bids_basename, bids_root, overwrite=True)
+    data = _from_tsv(participants_tsv)
+    with open(participants_json_fpath, 'r') as fin:
+        participants_json = json.load(fin)
+    assert 'subject_test_col1' in participants_json
+    assert data['age'][data['participant_id'].index('sub-01')] == '9'
+    assert data['subject_test_col1'][participant_idx] == 'S'
+    # in addition assert the original ordering of the new overwritten file
+    assert list(data.keys()) == orig_key_order
+
+    # if overwrite is False, then nothing should change from the above
+    with pytest.raises(FileExistsError, match='already exists'):
+        raw.info['subject_info'] = None
+        write_raw_bids(raw, bids_basename, bids_root, overwrite=False)
+    data = _from_tsv(participants_tsv)
+    with open(participants_json_fpath, 'r') as fin:
+        participants_json = json.load(fin)
+    assert 'subject_test_col1' in participants_json
+    assert data['age'][data['participant_id'].index('sub-01')] == '9'
+    assert data['subject_test_col1'][participant_idx] == 'S'
+    # in addition assert the original ordering of the new overwritten file
+    assert list(data.keys()) == orig_key_order
+
     # try and write preloaded data
     raw = mne.io.read_raw_fif(raw_fname, preload=True)
     with pytest.raises(ValueError, match='preloaded'):