Repository: bids-standard/pybids (python)

Issue:
X specification doesn't allow regex pattern matching/completion for variable names
Example use case:
```
...
 "Model": {
        "X": ["trials", "gain", "loss", "demeaned_RT", "rot_*", "trans_*", 1]
      },
...
```

Error message:

```
Traceback (most recent call last):
  File "/home/shank/miniconda3/envs/fitlins38_2/bin/fitlins", line 33, in <module>
    sys.exit(load_entry_point('fitlins', 'console_scripts', 'fitlins')())
  File "/home/shank/dev-packages/fitlins-local/fitlins/cli/run.py", line 434, in main
    sys.exit(run_fitlins(sys.argv[1:]))
  File "/home/shank/dev-packages/fitlins-local/fitlins/cli/run.py", line 427, in run_fitlins
    report_dict = build_report_dict(deriv_dir, work_dir, graph)
  File "/home/shank/dev-packages/fitlins-local/fitlins/viz/reports.py", line 68, in build_report_dict
    load_all_specs(all_specs, None, graph.root_node)
  File "/home/shank/dev-packages/fitlins-local/fitlins/utils/bids.py", line 46, in load_all_specs
    specs = node.run(group_by=node.group_by, force_dense=False)
  File "/home/shank/dev-packages/pybids/bids/modeling/statsmodels.py", line 460, in run
    node_output = BIDSStatsModelsNodeOutput(
  File "/home/shank/dev-packages/pybids/bids/modeling/statsmodels.py", line 604, in __init__
    raise ValueError("X specification includes variable(s) {}, but "
ValueError: X specification includes variable(s) ['trans_*', 'rot_*'], but these were not found in data matrix.
```

cc: @effigies 

Pull request:
ENH: Handle wildcards in model X
Fixes #795.

I can't easily see a good way to test this with our current testing infrastructure, so I think it'll need to wait.

Patch:
diff --git a/bids/modeling/statsmodels.py b/bids/modeling/statsmodels.py
index 5821d58b..aa3bef45 100644
--- a/bids/modeling/statsmodels.py
+++ b/bids/modeling/statsmodels.py
@@ -4,6 +4,8 @@ import json
 from collections import namedtuple, OrderedDict, Counter, defaultdict
 import itertools
 from functools import reduce
+import re
+import fnmatch
 
 import numpy as np
 import pandas as pd
@@ -520,6 +522,15 @@ class BIDSStatsModelsNode:
         return [c for c in self._collections if matches_entities(c, filters)]
 
 
+def expand_wildcards(selectors, pool):
+    out = list(selectors)
+    for spec in selectors:
+        if re.search(r'[\*\?\[\]]', spec):
+            idx = out.index(spec)
+            out[idx:idx + 1] = fnmatch.filter(pool, spec)
+    return out
+
+
 class BIDSStatsModelsNodeOutput:
     """Represents a single node in a BIDSStatsModelsGraph.
 
@@ -597,6 +608,8 @@ class BIDSStatsModelsNodeOutput:
             else:
                 var_names.append(int_name)
 
+        var_names = expand_wildcards(var_names, df.columns)
+
         # Verify all X names are actually present
         missing = list(set(var_names) - set(df.columns))
         if missing:
@@ -642,7 +655,9 @@ class BIDSStatsModelsNodeOutput:
 
             # Take the intersection of variables and Model.X (var_names), ignoring missing
             # variables (usually contrasts)
-            coll.variables = {v: coll.variables[v] for v in var_names if v in coll.variables}
+            coll.variables = {v: coll.variables[v]
+                              for v in expand_wildcards(var_names, coll.variables)
+                              if v in coll.variables}
             if not coll.variables:
                 continue
 


Test patch:
diff --git a/bids/modeling/tests/test_statsmodels.py b/bids/modeling/tests/test_statsmodels.py
index 312e4edf..55b9090a 100644
--- a/bids/modeling/tests/test_statsmodels.py
+++ b/bids/modeling/tests/test_statsmodels.py
@@ -10,7 +10,7 @@ import numpy as np
 import pytest
 
 from bids.modeling import BIDSStatsModelsGraph
-from bids.modeling.statsmodels import ContrastInfo
+from bids.modeling.statsmodels import ContrastInfo, expand_wildcards
 from bids.layout import BIDSLayout
 from bids.tests import get_test_data_path
 from bids.variables import BIDSVariableCollection
@@ -152,3 +152,20 @@ def test_entire_graph_smoketest(graph):
     assert model_spec.X.shape == (2, 1)
     assert model_spec.Z is None
     assert not set(model_spec.terms.keys()) - {"RT", "gain", "RT:gain"}
+
+
+def test_expand_wildcards():
+    # No wildcards == no modification
+    assert expand_wildcards(["a", "b"], ["a", "c"]) == ["a", "b"]
+    # No matches == removal
+    assert expand_wildcards(["a", "b*"], ["a", "c"]) == ["a"]
+    # Matches expand in-place
+    assert expand_wildcards(["a*", "b"], ["a", "c"]) == ["a", "b"]
+    assert expand_wildcards(["a*", "b"], ["a0", "c", "a1", "a2"]) == ["a0", "a1", "a2", "b"]
+    # Some examples
+    assert expand_wildcards(
+        ["trial_type.*"], ["trial_type.A", "trial_type.B"]
+    ) == ["trial_type.A", "trial_type.B"]
+    assert expand_wildcards(
+        ["non_steady_state*"], ["non_steady_state00", "non_steady_state01"]
+    ) == ["non_steady_state00", "non_steady_state01"]