Repository: mne-tools/mne-bids

Issue:
Replace pandas dependency with custom private functions
the pandas dependency is not used very much ... mainly for reading and writing csv files. And in some instances, replacing cells within a csv file.

We should be able to replace this dependency with a set of private functions.


Discussion:
given #99, I think we should maybe keep pandas - it would be too much of an unnecessary hassle to replace the functions with custom code. Other opinions?
after my work on #99 it actually made me think better of removing the pandas dependency as it can give us better control.
My suggestion is to essentially move to using numpy instead of pandas as numpy is already required and has functions for reading and writing tsv files.
I'll have a think and play with how simple replacing some of the pandas functions with numpy functions would be
you have my +100 and blessing for removing pandas. The less dependencies, the better!

Resolving patch:
diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py
--- a/mne_bids/mne_bids.py
+++ b/mne_bids/mne_bids.py
@@ -12,7 +12,6 @@
 import os.path as op
 
 import shutil as sh
-import pandas as pd
 from collections import defaultdict, OrderedDict
 
 import numpy as np
@@ -35,7 +34,7 @@
                     _infer_eeg_placement_scheme, _parse_bids_filename,
                     _handle_kind)
 from .io import _parse_ext, ALLOWED_EXTENSIONS, reader
-
+from mne_bids.tsv_handler import _from_tsv, _combine, _drop, _contains_row
 
 ALLOWED_KINDS = ['meg', 'eeg', 'ieeg']
 
@@ -115,9 +114,7 @@ def _channels_tsv(raw, fname, overwrite=False, verbose=True):
     _, ext = _parse_ext(raw.filenames[0], verbose=verbose)
     manufacturer = MANUFACTURERS[ext]
 
-    ignored_indexes = [raw.ch_names.index(ch_name) for ch_name in raw.ch_names
-                       if ch_name in
-                       IGNORED_CHANNELS.get(manufacturer, list())]
+    ignored_channels = IGNORED_CHANNELS.get(manufacturer, list())
 
     status, ch_type, description = list(), list(), list()
     for idx, ch in enumerate(raw.info['ch_names']):
@@ -137,18 +134,18 @@ def _channels_tsv(raw, fname, overwrite=False, verbose=True):
     n_channels = raw.info['nchan']
     sfreq = raw.info['sfreq']
 
-    df = pd.DataFrame(OrderedDict([
-                      ('name', raw.info['ch_names']),
-                      ('type', ch_type),
-                      ('units', units),
-                      ('low_cutoff', np.full((n_channels), low_cutoff)),
-                      ('high_cutoff', np.full((n_channels), high_cutoff)),
-                      ('description', description),
-                      ('sampling_frequency', np.full((n_channels), sfreq)),
-                      ('status', status)]))
-    df.drop(ignored_indexes, inplace=True)
+    ch_data = OrderedDict([
+        ('name', raw.info['ch_names']),
+        ('type', ch_type),
+        ('units', units),
+        ('low_cutoff', np.full((n_channels), low_cutoff)),
+        ('high_cutoff', np.full((n_channels), high_cutoff)),
+        ('description', description),
+        ('sampling_frequency', np.full((n_channels), sfreq)),
+        ('status', status)])
+    ch_data = _drop(ch_data, ignored_channels, 'name')
 
-    _write_tsv(fname, df, overwrite, verbose)
+    _write_tsv(fname, ch_data, overwrite, verbose)
 
     return fname
 
@@ -187,30 +184,27 @@ def _events_tsv(events, raw, fname, trial_type, overwrite=False,
     The function writes durations of zero for each event.
 
     """
-    # Start by filling all data that we know into a df
+    # Start by filling all data that we know into an ordered dictionary
     first_samp = raw.first_samp
     sfreq = raw.info['sfreq']
     events[:, 0] -= first_samp
 
-    data = OrderedDict([('onset', events[:, 0]),
+    # Onset column needs to be specified in seconds
+    data = OrderedDict([('onset', events[:, 0] / sfreq),
                         ('duration', np.zeros(events.shape[0])),
-                        ('trial_type', events[:, 2]),
+                        ('trial_type', None),
                         ('event_value', events[:, 2]),
                         ('event_sample', events[:, 0])])
 
-    df = pd.DataFrame.from_dict(data)
-
     # Now check if trial_type is specified or should be removed
     if trial_type:
         trial_type_map = {v: k for k, v in trial_type.items()}
-        df.trial_type = df.trial_type.map(trial_type_map)
+        data['trial_type'] = [trial_type_map.get(i, 'n/a') for
+                              i in events[:, 2]]
     else:
-        df.drop(labels=['trial_type'], axis=1, inplace=True)
+        del data['trial_type']
 
-    # Onset column needs to be specified in seconds
-    df.onset /= sfreq
-
-    _write_tsv(fname, df, overwrite, verbose)
+    _write_tsv(fname, data, overwrite, verbose)
 
     return fname
 
@@ -240,8 +234,10 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False,
 
     """
     subject_id = 'sub-' + subject_id
-    data = {'participant_id': [subject_id]}
+    data = OrderedDict(participant_id=[subject_id])
 
+    subject_age = "n/a"
+    sex = "n/a"
     subject_info = raw.info['subject_info']
     if subject_info is not None:
         sexes = {0: 'n/a', 1: 'M', 2: 'F'}
@@ -260,17 +256,15 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False,
         else:
             subject_age = "n/a"
 
-        data.update({'age': [subject_age], 'sex': [sex]})
-
-    df = pd.DataFrame(data=data,
-                      columns=['participant_id', 'age', 'sex'])
+    data.update({'age': [subject_age], 'sex': [sex]})
 
     if os.path.exists(fname):
-        orig_df = pd.read_csv(fname, sep='\t')
-        # whether the data exists identically in the current DataFrame
-        exact_included = df.values.tolist()[0] in orig_df.values.tolist()
-        # whether the subject id is in the existing DataFrame
-        sid_included = subject_id in orig_df['participant_id'].values
+        orig_data = _from_tsv(fname)
+        # whether the new data exists identically in the previous data
+        exact_included = _contains_row(orig_data,
+                                       [subject_id, subject_age, sex])
+        # whether the subject id is in the previous data
+        sid_included = subject_id in orig_data['participant_id']
         # if the subject data provided is different to the currently existing
         # data and overwrite is not True raise an error
         if (sid_included and not exact_included) and not overwrite:
@@ -278,16 +272,11 @@ def _participants_tsv(raw, subject_id, fname, overwrite=False,
                                   'list. Please set overwrite to '
                                   'True.' % subject_id)
         # otherwise add the new data
-        df = orig_df.append(df)
-        # and drop any duplicates as we want overwrite = True to force the old
-        # data to be overwritten
-        df.drop_duplicates(subset='participant_id', keep='last',
-                           inplace=True)
-        df = df.sort_values(by='participant_id')
+        data = _combine(orig_data, data, 'participant_id')
 
     # overwrite is forced to True as all issues with overwrite == False have
     # been handled by this point
-    _write_tsv(fname, df, True, verbose)
+    _write_tsv(fname, data, True, verbose)
 
     return fname
 
@@ -315,7 +304,7 @@ def _participants_json(fname, overwrite=False, verbose=True):
     cols['sex'] = {'Description': 'Biological sex of the participant',
                    'Levels': {'F': 'female', 'M': 'male'}}
 
-    _write_json(cols, fname, overwrite, verbose)
+    _write_json(fname, cols, overwrite, verbose)
 
     return fname
 
@@ -349,26 +338,21 @@ def _scans_tsv(raw, raw_fname, fname, overwrite=False, verbose=True):
     else:
         acq_time = 'n/a'
 
-    df = pd.DataFrame(data={'filename': ['%s' % raw_fname],
-                            'acq_time': [acq_time]},
-                      columns=['filename', 'acq_time'])
+    data = OrderedDict([('filename', ['%s' % raw_fname]),
+                        ('acq_time', [acq_time])])
 
     if os.path.exists(fname):
-        orig_df = pd.read_csv(fname, sep='\t')
+        orig_data = _from_tsv(fname)
         # if the file name is already in the file raise an error
-        if raw_fname in orig_df['filename'].values and not overwrite:
+        if raw_fname in orig_data['filename'] and not overwrite:
             raise FileExistsError('"%s" already exists in the scans list. '
                                   'Please set overwrite to True.' % raw_fname)
         # otherwise add the new data
-        df = orig_df.append(df)
-        # and drop any duplicates as we want overwrite = True to force the old
-        # data to be overwritten
-        df.drop_duplicates(subset='filename', keep='last', inplace=True)
-        df = df.sort_values(by='acq_time')
+        data = _combine(orig_data, data, 'filename')
 
     # overwrite is forced to True as all issues with overwrite == False have
     # been handled by this point
-    _write_tsv(fname, df, True, verbose)
+    _write_tsv(fname, data, True, verbose)
 
     return fname
 
@@ -425,7 +409,7 @@ def _coordsystem_json(raw, unit, orient, manufacturer, fname,
                 'HeadCoilCoordinateUnits': unit  # XXX validate this
                 }
 
-    _write_json(fid_json, fname, overwrite, verbose)
+    _write_json(fname, fid_json, overwrite, verbose)
 
     return fname
 
@@ -544,7 +528,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False,
     ch_info_json += ch_info_ch_counts
     ch_info_json = OrderedDict(ch_info_json)
 
-    _write_json(ch_info_json, fname, overwrite, verbose)
+    _write_json(fname, ch_info_json, overwrite, verbose)
 
     return fname
 
diff --git a/mne_bids/tsv_handler.py b/mne_bids/tsv_handler.py
new file mode 100644
--- /dev/null
+++ b/mne_bids/tsv_handler.py
@@ -0,0 +1,170 @@
+import numpy as np
+from collections import OrderedDict
+from copy import deepcopy
+
+
+def _combine(data1, data2, drop_column=None):
+    """Add two OrderedDict's together and optionally drop repeated data.
+
+    Parameters
+    ----------
+    data1 : collections.OrderedDict
+        Original OrderedDict.
+    data2 : collections.OrderedDict
+        New OrderedDict to be added to the original.
+    drop_column : str, optional
+        Name of the column to check for duplicate values in.
+        Any duplicates found will be dropped from the original data array (ie.
+        most recent value are kept).
+
+    Returns
+    -------
+    data : collections.OrderedDict
+        The new combined data.
+    """
+    data = deepcopy(data1)
+    for key, value in data2.items():
+        data[key].extend(value)
+    # Make sure that if there are any columns in data1 that didn't get new
+    # data they are populated with "n/a"'s.
+    for key in set(data1.keys()) - set(data2.keys()):
+        data[key].extend(["n/a"] * len(next(iter(data2.values()))))
+
+    if drop_column is None:
+        return data
+
+    # Find any repeated values and remove all but the most recent value.
+    n_rows = len(data[drop_column])
+    _, idxs = np.unique(data[drop_column][::-1], return_index=True)
+    for key in data:
+        data[key] = [data[key][n_rows - 1 - idx] for idx in idxs]
+
+    return data
+
+
+def _contains_row(data, row_data):
+    """Determines whether the specified row data exists in the OrderedDict.
+
+    Parameters
+    ----------
+    data : collections.OrderedDict
+        OrderedDict to check.
+    row_data : list
+        List of values to be searched for. This must match the contents of a
+        row exactly for a positive match to be returned.
+
+    Returns
+    -------
+    bool
+        True if `row_data` exists in `data`.
+    """
+    mask = None
+    for idx, column_data in enumerate(data.values()):
+        column_mask = np.in1d(np.array(column_data), row_data[idx])
+        mask = column_mask if mask is None else (mask & column_mask)
+    return np.any(mask)
+
+
+def _drop(data, values, column):
+    """Remove rows from the OrderedDict.
+
+    Parameters
+    ----------
+    data : collections.OrderedDict
+        Data to drop values from.
+    values : list
+        List of values to drop. Any row containing this value in the specified
+        column will be dropped.
+    column : string
+        Name of the column to check for the existence of `value` in.
+
+    Returns
+    -------
+    new_data : collections.OrderedDict
+        Copy of the original data with 0 or more rows dropped.
+    """
+    new_data = deepcopy(data)
+    mask = np.in1d(new_data[column], values, invert=True)
+    for key in new_data.keys():
+        new_data[key] = np.array(new_data[key])[mask].tolist()
+    return new_data
+
+
+def _from_tsv(fname, dtypes=None):
+    """Read a tsv file into an OrderedDict.
+
+    Parameters
+    ----------
+    fname : str
+        Path to the file being loaded.
+    dtypes : list, optional
+        List of types to cast the values loaded as. This is specified column by
+        column.
+        Defaults to None. In this case all the data is loaded as strings.
+
+    Returns
+    -------
+    data_dict : collections.OrderedDict
+        Keys are the column names, and values are the column data.
+    """
+    data = np.loadtxt(fname, dtype=str, delimiter='\t')
+    column_names = data[0, :]
+    info = data[1:, :]
+    data_dict = OrderedDict()
+    if dtypes is None:
+        dtypes = [str] * info.shape[1]
+    if not isinstance(dtypes, (list, tuple)):
+        dtypes = [dtypes] * info.shape[1]
+    if not len(dtypes) == info.shape[1]:
+        raise ValueError('dtypes length mismatch. Provided: {0}, '
+                         'Expected: {1}'.format(len(dtypes), info.shape[1]))
+    for i, name in enumerate(column_names):
+        data_dict[name] = info[:, i].astype(dtypes[i]).tolist()
+    return data_dict
+
+
+def _to_tsv(data, fname):
+    """Write an OrderedDict into a tsv file.
+
+    Parameters
+    ----------
+    data : collections.OrderedDict
+        Ordered dictionary containing data to be written to a tsv file.
+    fname : str
+        Path to the file being written.
+    """
+    n_rows = len(data[list(data.keys())[0]])
+    output = _tsv_to_str(data, n_rows)
+
+    with open(fname, 'w') as f:
+        f.write(output)
+
+
+def _tsv_to_str(data, rows=5):
+    """Return a string representation of the OrderedDict.
+
+    Parameters
+    ----------
+    data : collections.OrderedDict
+        OrderedDict to return string representation of.
+    rows : int, optional
+        Maximum number of rows of data to output.
+
+    Returns
+    -------
+    str
+        String representation of the first `rows` lines of `data`.
+    """
+    col_names = list(data.keys())
+    n_rows = len(data[col_names[0]])
+    output = list()
+    # write headings.
+    output.append('\t'.join(col_names))
+
+    # write column data.
+    max_rows = min(n_rows, rows)
+    for idx in range(max_rows):
+        row_data = list(str(data[key][idx]) for key in data)
+        output.append('\t'.join(row_data))
+
+    return '\n'.join(output)
diff --git a/mne_bids/utils.py b/mne_bids/utils.py
--- a/mne_bids/utils.py
+++ b/mne_bids/utils.py
@@ -22,6 +22,7 @@
 
 from .config import BIDS_VERSION
 from .io import _parse_ext
+from .tsv_handler import _to_tsv, _tsv_to_str
 
 
 def print_dir_tree(folder):
@@ -304,7 +305,7 @@ def make_dataset_description(path, name=None, data_license=None,
     pop_keys = [key for key, val in description.items() if val is None]
     for key in pop_keys:
         description.pop(key)
-    _write_json(description, fname, overwrite=True, verbose=verbose)
+    _write_json(fname, description, overwrite=True, verbose=verbose)
 
 
 def _age_on_date(bday, exp_date):
@@ -337,7 +338,7 @@ def _check_types(variables):
                              "Found type %s." % type(var))
 
 
-def _write_json(dictionary, fname, overwrite=False, verbose=False):
+def _write_json(fname, dictionary, overwrite=False, verbose=False):
     """Write JSON to a file."""
     if op.exists(fname) and not overwrite:
         raise FileExistsError('"%s" already exists. Please set '
@@ -353,16 +354,16 @@ def _write_json(dictionary, fname, overwrite=False, verbose=False):
         print(json_output)
 
 
-def _write_tsv(fname, df, overwrite=False, verbose=False):
-    """Write dataframe to a .tsv file."""
+def _write_tsv(fname, dictionary, overwrite=False, verbose=False):
+    """Write an ordered dictionary to a .tsv file"""
     if op.exists(fname) and not overwrite:
         raise FileExistsError('"%s" already exists. Please set '
                               'overwrite to True.' % fname)
-    df.to_csv(fname, sep='\t', index=False, na_rep='n/a', encoding='utf-8')
+    _to_tsv(dictionary, fname)
 
     if verbose:
         print(os.linesep + "Writing '%s'..." % fname + os.linesep)
-        print(df.head())
+        print(_tsv_to_str(dictionary))
 
 
 def _check_key_val(key, val):


Test patch:
diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py
--- a/mne_bids/tests/test_mne_bids.py
+++ b/mne_bids/tests/test_mne_bids.py
@@ -18,14 +18,13 @@
 
 from datetime import datetime
 
-import pandas as pd
-
 import mne
 from mne.datasets import testing
 from mne.utils import _TempDir, run_subprocess
 from mne.io.constants import FIFF
 
 from mne_bids import make_bids_basename, make_bids_folders, write_raw_bids
+from mne_bids.tsv_handler import _from_tsv
 
 base_path = op.join(op.dirname(mne.__file__), 'io')
 subject_id = '01'
@@ -175,8 +174,8 @@ def test_kit():
         subject=subject_id, session=session_id, task=task, run=run,
         suffix='channels.tsv', acquisition=acq,
         prefix=op.join(output_path, 'sub-01/ses-01/meg'))
-    df = pd.read_csv(channels_tsv, sep='\t')
-    assert not ('STI 014' in df['name'].values)
+    data = _from_tsv(channels_tsv)
+    assert 'STI 014' not in data['name']
 
     # ensure the marker file is produced in the right place
     raw_folder = make_bids_basename(
@@ -251,9 +250,9 @@ def test_vhdr():
     channels_tsv_name = op.join(output_path, 'sub-' + subject_id,
                                 'ses-' + session_id, 'eeg',
                                 bids_basename + '_channels.tsv')
-    df = pd.read_csv(channels_tsv_name, sep='\t', keep_default_na=False)
-    assert df.loc[df['name'] == 'FP1', 'units'].all() == 'µV'
-    assert df.loc[df['name'] == 'CP5', 'units'].all() == 'n/a'
+    data = _from_tsv(channels_tsv_name)
+    assert data['units'][data['name'].index('FP1')] == 'µV'
+    assert data['units'][data['name'].index('CP5')] == 'n/a'
 
     # create another bids folder with the overwrite command and check
     # no files are in the folder
@@ -301,15 +300,15 @@ def test_edf():
         subject=subject_id, session=session_id, task=task, run=run,
         suffix='channels.tsv', acquisition=acq,
         prefix=op.join(output_path, 'sub-01/ses-01/eeg'))
-    df = pd.read_csv(channels_tsv, sep='\t')
-    assert 'ElectroMyoGram' in df['description'].values
+    data = _from_tsv(channels_tsv)
+    assert 'ElectroMyoGram' in data['description']
 
     # check that the scans list contains two scans
     scans_tsv = make_bids_basename(
         subject=subject_id, session=session_id, suffix='scans.tsv',
         prefix=op.join(output_path, 'sub-01/ses-01'))
-    df = pd.read_csv(scans_tsv, sep='\t')
-    assert df.shape[0] == 2
+    data = _from_tsv(scans_tsv)
+    assert len(list(data.values())[0]) == 2
 
     # Also cover iEEG
     # We use the same data and pretend that eeg channels are ecog
diff --git a/mne_bids/tests/test_pick.py b/mne_bids/tests/test_pick.py
--- a/mne_bids/tests/test_pick.py
+++ b/mne_bids/tests/test_pick.py
@@ -1,4 +1,7 @@
 """Test for the coil type picking function"""
+# Authors: Matt Sanderson <matt.sanderson@mq.edu.au>
+#
+# License: BSD (3-clause)
 
 import os.path as op
 
diff --git a/mne_bids/tests/test_tsv_handler.py b/mne_bids/tests/test_tsv_handler.py
new file mode 100644
--- /dev/null
+++ b/mne_bids/tests/test_tsv_handler.py
@@ -0,0 +1,58 @@
+"""Test for the tsv handling functions"""
+# Authors: Matt Sanderson <matt.sanderson@mq.edu.au>
+#
+# License: BSD (3-clause)
+
+import os.path as op
+from collections import OrderedDict as odict
+from mne_bids.tsv_handler import (_from_tsv, _to_tsv, _combine, _drop,
+                                  _contains_row, _tsv_to_str)
+import pytest
+
+from mne.utils import _TempDir
+
+
+def test_tsv_handler():
+    # create some dummy data
+    d = odict(a=[1, 2, 3, 4], b=['five', 'six', 'seven', 'eight'])
+    assert _contains_row(d, [1, 'five'])
+    d2 = odict(a=[5], b=['nine'])
+    d = _combine(d, d2)
+    assert 5 in d['a']
+    d2 = odict(a=[5])
+    d = _combine(d, d2)
+    assert 'n/a' in d['b']
+    d2 = odict(a=[5], b=['ten'])
+    d = _combine(d, d2, drop_column='a')
+    # make sure that the repeated data was dropped
+    assert 'nine' not in d['b']
+    print(_tsv_to_str(d))
+
+    tempdir = _TempDir()
+    d_path = op.join(tempdir, 'output.tsv')
+
+    # write the data to an output tsv file
+    _to_tsv(d, d_path)
+    # now read it back
+    d = _from_tsv(d_path)
+    # test reading the file in with the incorrect number of datatypes raises
+    # an Error
+    with pytest.raises(ValueError):
+        d = _from_tsv(d_path, dtypes=[str])
+    # we can also pass just a single data type and it will be applied to all
+    # columns
+    d = _from_tsv(d_path, str)
+
+    # remove any rows with 2 or 5 in them
+    d = _drop(d, [2, 5], 'a')
+    assert 2 not in d['a']
+
+    # test combining data with differing numbers of columns
+    d = odict(a=[1, 2], b=['three', 'four'])
+    d2 = odict(a=[4], b=['five'], c=[3.1415])
+    # raise error if a new column is tried to be added
+    with pytest.raises(KeyError):
+        d = _combine(d, d2)
+    d2 = odict(a=[5])
+    d = _combine(d, d2)
+    assert d['b'] == ['three', 'four', 'n/a']