Repository: bids-standard/pybids
Pull request #495 (2019-09-13 20:10:30)

Fix default ignore
For unclear reasons (i.e., human error), the `_default_ignore` list was being... ignored by default (🥁). This patch ensures that passing `ignore=None` will ignore directories like `sourcedata/` and `code/`, and also that specifying `ignore` explicitly overwrites the defaults.


Patch:
diff --git a/bids/layout/index.py b/bids/layout/index.py
index 7ec20e86..f51b69d1 100644
--- a/bids/layout/index.py
+++ b/bids/layout/index.py
@@ -205,7 +205,12 @@ class BIDSLayoutIndexer(object):
 
                 if ext == 'json':
                     with open(bf.path, 'r') as handle:
-                        payload = json.load(handle)
+                        try:
+                            payload = json.load(handle)
+                        except json.JSONDecodeError as e:
+                            msg = ("Error occurred while trying to decode JSON"
+                                   " from file '{}'.".format(bf.path))
+                            six.raise_from(IOError(msg), e)
                 else:
                     payload = None
 
diff --git a/bids/layout/layout.py b/bids/layout/layout.py
index b00336ea..d007f3f0 100644
--- a/bids/layout/layout.py
+++ b/bids/layout/layout.py
@@ -132,7 +132,9 @@ class BIDSLayout(object):
             absolute path, or be relative to the BIDS project root. If an
             SRE_Pattern is passed, the contained regular expression will be
             matched against the full (absolute) path of all files and
-            directories.
+            directories. By default, indexing ignores all files in 'code/',
+            'stimuli/', 'sourcedata/', 'models/', and any hidden files/dirs
+            beginning with '.' at root level.
         force_index (str, SRE_Pattern, list): Path(s) to forcibly index in the
             BIDSLayout, even if they would otherwise fail validation. See the
             documentation for the ignore argument for input format details.
@@ -161,8 +163,8 @@ class BIDSLayout(object):
             indexing will be faster).
     """
 
-    _default_ignore = {"code", "stimuli", "sourcedata", "models",
-                       "derivatives", re.compile(r'^\.')}
+    _default_ignore = ("code", "stimuli", "sourcedata", "models",
+                       re.compile(r'^\.'))
 
     def __init__(self, root, validate=True, absolute_paths=True,
                  derivatives=False, config=None, sources=None, ignore=None,
@@ -184,6 +186,9 @@ class BIDSLayout(object):
         # Do basic BIDS validation on root directory
         self._validate_root()
 
+        if ignore is None:
+            ignore = self._default_ignore
+
         # Instantiate after root validation to ensure os.path.join works
         self.ignore = [os.path.abspath(os.path.join(self.root, patt))
                        if isinstance(patt, six.string_types) else patt


Test patch:
diff --git a/bids/layout/tests/test_layout.py b/bids/layout/tests/test_layout.py
index 1faa7132..1277055b 100644
--- a/bids/layout/tests/test_layout.py
+++ b/bids/layout/tests/test_layout.py
@@ -247,6 +247,23 @@ def test_get_return_sorted(layout_7t_trt):
     assert files == paths
 
 
+def test_ignore_files(layout_ds005):
+    data_dir = join(get_test_data_path(), 'ds005')
+    target1 = join(data_dir, 'models', 'ds-005_type-test_model.json')
+    target2 = join(data_dir, 'models', 'extras', 'ds-005_type-test_model.json')
+    layout1 = BIDSLayout(data_dir, validate=False)
+    assert target1 not in layout_ds005.files
+    assert target1 not in layout1.files
+    assert target2 not in layout1.files
+    # now the models/ dir should show up, because passing ignore explicitly
+    # overrides the default - but 'model/extras/' should still be ignored because
+    # of the regex.
+    ignore = [re.compile('xtra'), 'dummy']
+    layout2 = BIDSLayout(data_dir, validate=False, ignore=ignore)
+    assert target1 in layout2.files
+    assert target2 not in layout2.files
+
+
 def test_force_index(layout_ds005):
     data_dir = join(get_test_data_path(), 'ds005')
     target= join(data_dir, 'models', 'ds-005_type-test_model.json')
@@ -570,4 +587,4 @@ def test_get_with_wrong_dtypes(layout_7t_trt):
     assert (l.get(run=1) == l.get(run='1') == l.get(run=np.int64(1)) ==
             l.get(run=[1, '15']))
     assert not l.get(run='not_numeric')
-    assert l.get(session=1) == l.get(session='1')
\ No newline at end of file
+    assert l.get(session=1) == l.get(session='1')