Repository: bids-standard/pybids
Pull request #325 (2018-12-17 17:48:09)

Adds DropNA transformation
Adds a DropNA transformation, per the draft specification. This simply drops all rows with missing values from the named variable(s).


Patch:
diff --git a/bids/analysis/analysis.py b/bids/analysis/analysis.py
index d082ae67..fd3c2866 100644
--- a/bids/analysis/analysis.py
+++ b/bids/analysis/analysis.py
@@ -59,7 +59,7 @@ class Analysis(object):
             step = Step(self.layout, index=i, **step_args)
             self.steps.append(step)
 
-    def setup(self, steps=None, agg_func='mean', **kwargs):
+    def setup(self, steps=None, drop_na=False, **kwargs):
         ''' Set up the sequence of steps for analysis.
 
         Args:
@@ -68,14 +68,9 @@ class Analysis(object):
                 JSON config block list, or a str giving the (unique) name of
                 the step, as specified in the JSON config. Steps that do not
                 match either index or name will be skipped.
-            agg_func (str or Callable): The aggregation function to use when
-                combining rows from the previous level of analysis. E.g.,
-                when analyzing a 'subject'-level block, inputs coming from the
-                'session' level are typically averaged to produce individual
-                subject-level estimates. Must be either a string giving the
-                name of a function recognized by apply() in pandas, or a
-                Callable that takes a DataFrame as input and returns a Series
-                or a DataFrame. NOTE: CURRENTLY UNIMPLEMENTED.
+            drop_na (bool): Boolean indicating whether or not to automatically
+                drop events that have a n/a amplitude when reading in data
+                from event files.
         '''
 
         # In the beginning, there was nothing
@@ -91,7 +86,7 @@ class Analysis(object):
             if steps is not None and i not in steps and b.name not in steps:
                 continue
 
-            b.setup(input_nodes, **selectors)
+            b.setup(input_nodes, drop_na=drop_na, **selectors)
             input_nodes = b.output_nodes
 
 
@@ -170,13 +165,16 @@ class Step(object):
         entities = pd.concat(entities, axis=1, sort=True).T
         return BIDSVariableCollection.from_df(data, entities, self.level)
 
-    def setup(self, input_nodes=None, **kwargs):
+    def setup(self, input_nodes=None, drop_na=False, **kwargs):
         ''' Set up the Step and construct the design matrix.
 
         Args:
             input_nodes (list): Optional list of Node objects produced by
                 the preceding Step in the analysis. If None, uses any inputs
                 passed in at Step initialization.
+            drop_na (bool): Boolean indicating whether or not to automatically
+                drop events that have a n/a amplitude when reading in data
+                from event files.
             kwargs: Optional keyword arguments to pass onto load_variables.
         '''
         self.output_nodes = []
@@ -188,7 +186,8 @@ class Step(object):
             kwargs = kwargs.copy()
             kwargs.pop('scan_length', None)
 
-        collections = self.layout.get_collections(self.level, **kwargs)
+        collections = self.layout.get_collections(self.level, drop_na=drop_na,
+                                                  **kwargs)
         objects = collections + input_nodes
 
         objects, kwargs = self._filter_objects(objects, kwargs)
diff --git a/bids/analysis/transformations/__init__.py b/bids/analysis/transformations/__init__.py
index 9d874f9a..2abaf33f 100644
--- a/bids/analysis/transformations/__init__.py
+++ b/bids/analysis/transformations/__init__.py
@@ -1,7 +1,7 @@
 from .compute import (Sum, Product, Scale, Orthogonalize, Threshold, And, Or,
                       Not, Demean, Convolve)
 from .munge import (Split, Rename, Assign, Copy, Factor, Filter, Select,
-                    Delete, Replace, ToDense)
+                    Delete, DropNA, Replace, ToDense)
 
 __all__ = [
     'And',
@@ -10,6 +10,7 @@ __all__ = [
     'Copy',
     'Demean',
     'Delete',
+    'DropNA',
     'Factor',
     'Filter',
     'Not',
diff --git a/bids/analysis/transformations/munge.py b/bids/analysis/transformations/munge.py
index 25e67b75..3468af7c 100644
--- a/bids/analysis/transformations/munge.py
+++ b/bids/analysis/transformations/munge.py
@@ -100,6 +100,21 @@ class Delete(Transformation):
                                      if k not in variables}
 
 
+class DropNA(Transformation):
+
+    _groupable = False
+    _input_type = 'variable'
+    _return_type = 'variable'
+    _allow_categorical = ('variables',)
+
+    def _transform(self, var):
+        
+        # Identify non-NA rows
+        valid = var.values.notna().values
+        var.select_rows(valid)
+        return var
+
+
 class Factor(Transformation):
 
     _groupable = False
@@ -173,9 +188,7 @@ class Filter(Transformation):
         data = data.query(query)
 
         # Truncate target variable to retained rows
-        var.onset = var.onset[data.index]
-        var.duration = var.duration[data.index]
-        var.values = var.values.iloc[data.index]
+        var.select_rows(data.index.values)
 
         return var
 
diff --git a/bids/variables/variables.py b/bids/variables/variables.py
index bc28c7eb..423b8d8e 100644
--- a/bids/variables/variables.py
+++ b/bids/variables/variables.py
@@ -267,6 +267,19 @@ class SimpleVariable(BIDSVariable):
         data = data.rename(columns={name: 'amplitude'})
         return cls(name, data, source=variables[0].source, **kwargs)
 
+    def select_rows(self, rows):
+        ''' Truncate internal arrays to keep only the specified rows.
+
+        Args:
+            rows (array): An integer or boolean array identifying the indices
+                of rows to keep.
+        '''
+        self.values = self.values.iloc[rows]
+        self.index = self.index.iloc[rows, :]
+        for prop in self._property_columns:
+            vals = getattr(self, prop)[rows]
+            setattr(self, prop, vals)
+
 
 class SparseRunVariable(SimpleVariable):
     ''' A sparse representation of a single column of events.


Test patch:
diff --git a/bids/analysis/tests/test_transformations.py b/bids/analysis/tests/test_transformations.py
index 46c37400..9db5625f 100644
--- a/bids/analysis/tests/test_transformations.py
+++ b/bids/analysis/tests/test_transformations.py
@@ -1,5 +1,8 @@
 # from bids.analysis.variables import load_variables
 from bids.analysis import transformations as transform
+from bids.variables import SparseRunVariable
+from bids.variables.entities import RunInfo
+from bids.variables.kollekshuns import BIDSRunVariableCollection
 from bids.layout import BIDSLayout
 import pytest
 from os.path import join, sep
@@ -18,6 +21,18 @@ def collection():
     return collection
 
 
+@pytest.fixture
+def sparse_run_variable_with_missing_values():
+    data = pd.DataFrame({
+        'onset': [2, 5, 11, 17],
+        'duration': [1.2, 1.6, 0.8, 2],
+        'amplitude': [1, 1, np.nan, 1]
+    })
+    run_info = [RunInfo({'subject': '01'}, 20, 2, 'dummy.nii.gz')]
+    var = SparseRunVariable('var', data, run_info, 'events')
+    return BIDSRunVariableCollection([var])
+
+
 def test_rename(collection):
     dense_rt = collection.variables['RT'].to_dense(10)
     assert len(dense_rt.values) == 230400
@@ -52,7 +67,6 @@ def test_sum(collection):
                       output='sum', weights=[1, 1, 1])
 
 
-
 def test_scale(collection):
     transform.Scale(collection, variables=['RT', 'parametric gain'],
                     output=['RT_Z', 'gain_Z'], groupby=['run', 'subject'])
@@ -362,3 +376,15 @@ def test_not(collection):
     transform.Not(coll, 'RT')
     post_rt = coll.variables['RT'].values.values
     assert (post_rt == ~pre_rt.astype(bool)).all()
+
+
+def test_dropna(sparse_run_variable_with_missing_values):
+    var = sparse_run_variable_with_missing_values.variables['var']
+    coll = sparse_run_variable_with_missing_values.clone()
+    transform.DropNA(coll, 'var')
+    post_trans = coll.variables['var']
+    assert len(var.values) > len(post_trans.values)
+    assert np.array_equal(post_trans.values, [1, 1, 1])
+    assert np.array_equal(post_trans.onset, [2, 5, 17])
+    assert np.array_equal(post_trans.duration, [1.2, 1.6, 2])
+    assert len(post_trans.index) == 3