Repository: grabbles/grabbit

Issue:
Enable coveralls

Discussion:

Patch:
diff --git a/.coveragerc b/.coveragerc
index a72e893..777a95e 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,5 +1,5 @@
 [run]
-source =
-	grabbit/
+include = grabbit/
 omit =
-    */tests/*
+	*/tests/*
+	*/external/*
diff --git a/.travis.yml b/.travis.yml
index 57590d2..96455a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,9 +12,7 @@ install:
   - pip install -e '.'
 
 script:
-  - PYTHONPATH=$PWD coverage run `which py.test` grabbit
-  - py.test --cov-report term-missing --cov=grabbit
-  - cd examples; PYTHONPATH=.. runipy *.ipynb  # test example notebooks to run and not fail
+  - py.test --pyargs grabbit --cov-report term-missing --cov=grabbit
 
 after_success:
   - coveralls
diff --git a/grabbit/core.py b/grabbit/core.py
index 9cb60dc..6f5ba4e 100644
--- a/grabbit/core.py
+++ b/grabbit/core.py
@@ -122,29 +122,50 @@ class File(object):
 class Domain(object):
 
     def __init__(self, name, config, root):
+        """
+        A set of rules that applies to one or more directories
+        within a Layout.
+
+        Args:
+            name (str): The name of the Domain.
+            config (dict): The configuration dictionary that defines the
+                entities and paths for the current domain.
+            root (str, list): The root directory or directories to which the
+                Domain's rules applies. Can be either a single path, or a list.
+        """
 
         self.name = name
         self.config = config
         self.root = root
         self.entities = {}
         self.files = []
-        self.filtering_regex = {}
         self.path_patterns = []
 
-        if 'index' in config:
-            self.filtering_regex = config['index']
-            if self.filtering_regex.get('include') and \
-               self.filtering_regex.get('exclude'):
-                raise ValueError("You can only define either include or "
-                                 "exclude regex, not both.")
+        self.include = listify(self.config.get('include', []))
+        self.exclude = listify(self.config.get('exclude', []))
+
+        if self.include and self.exclude:
+            raise ValueError("The 'include' and 'exclude' arguments cannot "
+                             "both be set. Please pass at most one of these "
+                             "for domain '%s'." % self.name)
 
         if 'default_path_patterns' in config:
             self.path_patterns += listify(config['default_path_patterns'])
 
     def add_entity(self, ent):
+        ''' Add an Entity.
+
+        Args:
+            ent (Entity): The Entity to add.
+        '''
         self.entities[ent.name] = ent
 
     def add_file(self, file):
+        ''' Add a file to tracking.
+
+        Args:
+            file (File): The File to add to tracking.
+        '''
         self.files.append(file)
 
 
@@ -281,16 +302,20 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
     def __init__(self, path, config=None, index=None, dynamic_getters=False,
                  absolute_paths=True, regex_search=False, entity_mapper=None,
-                 path_patterns=None, config_filename='layout.json'):
+                 path_patterns=None, config_filename='layout.json',
+                 include=None, exclude=None):
         """
         A container for all the files and metadata found at the specified path.
 
         Args:
             path (str): The root path of the layout.
-            config (str, list): The path to the JSON config file that defines
-                the entities and paths for the current layout. If a list is
-                provided, treat as several paths to config files, creating
-                one master config with all of them merged (in order).
+            config (str, list, dict): A specification of the configuration
+                file(s) defining domains to use in the Layout. Must be one of:
+
+                - A dictionary containing config information
+                - A string giving the path to a JSON file containing the config
+                - A list, where each element is one of the above
+
             index (str): Optional path to a saved index file. If a valid value
                 is passed, this index is used to populate Files and Entities,
                 and the normal indexing process (which requires scanning all
@@ -327,8 +352,20 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
                 Every directory will be scanned for this file, and if found,
                 the config file will be read in and added to the list of
                 configs.
+            include (str, list): A string or list specifying regexes used to
+                globally filter files when indexing. A file or directory
+                *must* match at least of the passed values in order to be
+                retained in the index. Cannot be used together with 'exclude'.
+            exclude (str, list): A string or list specifying regexes used to
+                globally filter files when indexing. If a file or directory
+                *must* matches any of the passed values, it will be dropped
+                from indexing. Cannot be used together with 'include'.
         """
 
+        if include is not None and exclude is not None:
+            raise ValueError("You cannot specify both the include and exclude"
+                             " arguments. Please pass at most one of these.")
+
         self.root = abspath(path) if absolute_paths else path
         self.entities = OrderedDict()
         self.files = {}
@@ -339,6 +376,8 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         self.path_patterns = path_patterns if path_patterns else []
         self.config_filename = config_filename
         self.domains = OrderedDict()
+        self.include = listify(include or [])
+        self.exclude = listify(exclude or [])
 
         if config is not None:
             for c in listify(config):
@@ -413,7 +452,8 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
         filename = f if isinstance(f, six.string_types) else f.path
 
-        if os.path.isabs(filename) and filename.startswith(self.root + os.path.sep):
+        if os.path.isabs(filename) and filename.startswith(
+                self.root + os.path.sep):
             # for filenames under the root - analyze relative path to avoid
             # bringing injustice to the grandkids of some unfortunately named
             # root directories.
@@ -422,19 +462,20 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         if domains is None:
             domains = list(self.domains.keys())
 
+        domains = [self.domains[dom] for dom in domains]
+
+        # Inject the Layout at the first position for global include/exclude
+        domains.insert(0, self)
         for dom in domains:
-            dom = self.domains[dom]
             # If file matches any include regex, then True
-            include_regex = dom.filtering_regex.get('include', [])
-            if include_regex:
-                for regex in include_regex:
+            if dom.include:
+                for regex in dom.include:
                     if re.match(regex, filename):
-                        break
-                else:
-                    return False
+                        return True
+                return False
             else:
-                # If file matches any excldue regex, then false
-                for regex in dom.filtering_regex.get('exclude', []):
+                # If file matches any exclude regex, then False
+                for regex in dom.exclude:
                     if re.match(regex, filename, flags=re.UNICODE):
                         return False
 
@@ -473,7 +514,13 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
     def _get_domains_for_file(self, f):
         if isinstance(f, File):
             return f.domains
-        return [d.name for d in self.domains.values() if f.startswith(d.root)]
+        domains = []
+        for d in self.domains.values():
+            for path in listify(d.root):
+                if f.startswith(path):
+                    domains.append(d.name)
+                    break
+        return domains
 
     def _index_file(self, root, f, domains=None, update_layout=True):
 
@@ -502,8 +549,8 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
         # Only keep Files that match at least one Entity, and all
         # mandatory Entities
-        if update_layout and file_ents and not (self.mandatory
-                                                - set(file_ents)):
+        if update_layout and file_ents and not (self.mandatory -
+                                                set(file_ents)):
             self.files[f.path] = f
             # Bind the File to all of the matching entities
             for name, tag in f.tags.items():
@@ -552,12 +599,13 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
             if self.config_filename in filenames:
                 config_path = os.path.join(root, self.config_filename)
                 config = json.load(open(config_path, 'r'))
-                self._load_domain(config)
+                root = config.get('root', root)
+                self._load_domain(config, root=root)
 
                 # Filter Domains if current dir's config file has an
                 # include directive
-                if 'include' in config:
-                    missing = set(config['include']) - set(domains)
+                if 'domains' in config:
+                    missing = set(config['domains']) - set(domains)
                     if missing:
                         msg = ("Missing configs '%s' specified in include "
                                "directive of config '%s'. Please make sure "
@@ -565,7 +613,7 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
                                "directory %s.") % (missing, config['name'],
                                                    root)
                         raise ValueError(msg)
-                    domains = config['include']
+                    domains = config['domains']
                 domains.append(config['name'])
 
                 filenames.remove(self.config_filename)


Test patch:
diff --git a/grabbit/tests/specs/test.json b/grabbit/tests/specs/test.json
index e0c6630..39ed11a 100644
--- a/grabbit/tests/specs/test.json
+++ b/grabbit/tests/specs/test.json
@@ -1,8 +1,6 @@
 {
   "name": "test",
-  "index" : {
-    "exclude" : [".*derivatives.*"]
-  },
+  "exclude" : [".*derivatives.*"],
   "entities": [
     {
       "name": "subject",
diff --git a/grabbit/tests/specs/test_include.json b/grabbit/tests/specs/test_include.json
index f3a87ef..8741d83 100644
--- a/grabbit/tests/specs/test_include.json
+++ b/grabbit/tests/specs/test_include.json
@@ -1,8 +1,6 @@
 {
   "name": "test_with_includes",
-  "index" : {
-    "include" : ["sub-(\\d+)", "ses-.*", "func", "fmap", ".*\\..*"]
-  },
+  "include" : ["sub-(\\d+)", "ses-.*", "func", "fmap", ".*\\..*"],
   "entities": [
     {
       "name": "subject",
diff --git a/grabbit/tests/specs/test_with_mapper.json b/grabbit/tests/specs/test_with_mapper.json
index 84c3d28..20fca11 100644
--- a/grabbit/tests/specs/test_with_mapper.json
+++ b/grabbit/tests/specs/test_with_mapper.json
@@ -1,8 +1,6 @@
 {
   "name": "test_with_mapper",
-  "index" : {
-    "exclude" : [".*derivatives.*"]
-  },
+  "exclude" : [".*derivatives.*"],
   "entities": [
     {
       "name": "subject",
diff --git a/grabbit/tests/test_core.py b/grabbit/tests/test_core.py
index c0087ee..e85cbe8 100644
--- a/grabbit/tests/test_core.py
+++ b/grabbit/tests/test_core.py
@@ -1,6 +1,7 @@
 import pytest
 from grabbit import File, Entity, Layout, Tag, merge_layouts
 import os
+from os.path import join
 import posixpath as psp
 import tempfile
 import json
@@ -14,17 +15,17 @@ def file(tmpdir):
     testfile = 'sub-03_ses-2_task-rest_acq-fullbrain_run-2_bold.nii.gz'
     fn = tmpdir.mkdir("tmp").join(testfile)
     fn.write('###')
-    return File(os.path.join(str(fn)))
+    return File(join(str(fn)))
 
 
 @pytest.fixture(scope='module', params=['local', 'hdfs'])
 def bids_layout(request):
     if request.param == 'local':
-        root = os.path.join(DIRNAME, 'data', '7t_trt')
+        root = join(DIRNAME, 'data', '7t_trt')
         # note about test.json:
         # in this test.json 'subject' regex was left to contain possible
         # leading 0; the other fields (run, session) has leading 0 stripped
-        config = os.path.join(DIRNAME, 'specs', 'test.json')
+        config = join(DIRNAME, 'specs', 'test.json')
         return Layout(root, config, regex_search=True)
     else:
         hdfs = pytest.importorskip("hdfs")
@@ -39,15 +40,15 @@ def bids_layout(request):
 
 @pytest.fixture(scope='module')
 def stamp_layout():
-    root = os.path.join(DIRNAME, 'data', 'valuable_stamps')
-    config = os.path.join(DIRNAME, 'specs', 'stamps.json')
+    root = join(DIRNAME, 'data', 'valuable_stamps')
+    config = join(DIRNAME, 'specs', 'stamps.json')
     return Layout(root, config, config_filename='dir_config.json')
 
 
 @pytest.fixture(scope='module')
 def layout_include(request):
-    root = os.path.join(DIRNAME, 'data', '7t_trt')
-    config = os.path.join(DIRNAME, 'specs', 'test_include.json')
+    root = join(DIRNAME, 'data', '7t_trt')
+    config = join(DIRNAME, 'specs', 'test_include.json')
     return Layout(root, config, regex_search=True)
 
 
@@ -99,7 +100,7 @@ class TestEntity:
     def test_matches(self, tmpdir):
         filename = "aardvark-4-reporting-for-duty.txt"
         tmpdir.mkdir("tmp").join(filename).write("###")
-        f = File(os.path.join(str(tmpdir), filename))
+        f = File(join(str(tmpdir), filename))
         e = Entity('avaricious', 'aardvark-(\d+)')
         e.matches(f, update_file=True)
         assert 'avaricious' in f.entities
@@ -134,15 +135,37 @@ class TestLayout:
         assert isinstance(bids_layout.mandatory, set)
         assert not bids_layout.dynamic_getters
 
+    def test_init_with_include_arg(self, bids_layout):
+        root = join(DIRNAME, 'data', '7t_trt')
+        config = join(DIRNAME, 'specs', 'test.json')
+        layout = Layout(root, config, regex_search=True, include='sub-\d*')
+        target = join(root, "dataset_description.json")
+        assert target in bids_layout.files
+        assert target not in layout.files
+        assert join(root, "sub-01", "sub-01_sessions.tsv") in layout.files
+        with pytest.raises(ValueError):
+            layout = Layout(root, config, include='sub-\d*', exclude="meh")
+
+    def test_init_with_exclude_arg(self, bids_layout):
+        root = join(DIRNAME, 'data', '7t_trt')
+        config = join(DIRNAME, 'specs', 'test.json')
+        layout = Layout(root, config, regex_search=True, exclude='sub-\d*')
+        target = join(root, "dataset_description.json")
+        assert target in bids_layout.files
+        assert target in layout.files
+        sub_file = join(root, "sub-01", "sub-01_sessions.tsv")
+        assert sub_file in bids_layout.files
+        assert sub_file not in layout.files
+
     def test_absolute_paths(self, bids_layout):
         result = bids_layout.get(subject=1, run=1, session=1)
         assert result  # that we got some entries
         assert all([os.path.isabs(f.filename) for f in result])
 
         if not hasattr(bids_layout, '_hdfs_client'):
-            root = os.path.join(DIRNAME, 'data', '7t_trt')
+            root = join(DIRNAME, 'data', '7t_trt')
             root = os.path.relpath(root)
-            config = os.path.join(DIRNAME, 'specs', 'test.json')
+            config = join(DIRNAME, 'specs', 'test.json')
 
             layout = Layout(root, config, absolute_paths=False)
 
@@ -174,8 +197,8 @@ class TestLayout:
             assert all([os.path.isabs(f.filename) for f in result])
 
     @pytest.mark.parametrize('data_dir, config',
-                             [(os.path.join(DIRNAME, 'data', '7t_trt'),
-                               os.path.join(DIRNAME, 'specs', 'test.json')),
+                             [(join(DIRNAME, 'data', '7t_trt'),
+                               join(DIRNAME, 'specs', 'test.json')),
                               (psp.join('hdfs://localhost:9000/grabbit/test/',
                                'data', '7t_trt'),
                                psp.join('hdfs://localhost:9000/grabbit/test/',
@@ -244,7 +267,7 @@ class TestLayout:
         nearest = bids_layout.get_nearest(
             result, type='sessions', extensions='tsv',
             ignore_strict_entities=['type'])
-        target = os.path.join('7t_trt', 'sub-01', 'sub-01_sessions.tsv')
+        target = join('7t_trt', 'sub-01', 'sub-01_sessions.tsv')
         assert target in nearest
         nearest = bids_layout.get_nearest(
             result, extensions='tsv', all_=True,
@@ -257,9 +280,9 @@ class TestLayout:
         assert nearest[0].subject == '01'
 
     def test_index_regex(self, bids_layout, layout_include):
-        targ = os.path.join(bids_layout.root, 'derivatives', 'excluded.json')
+        targ = join(bids_layout.root, 'derivatives', 'excluded.json')
         assert targ not in bids_layout.files
-        targ = os.path.join(layout_include.root, 'models',
+        targ = join(layout_include.root, 'models',
                             'excluded_model.json')
         assert targ not in layout_include.files
 
@@ -284,13 +307,13 @@ class TestLayout:
         os.unlink(tmp)
 
     def test_load_index(self, bids_layout):
-        f = os.path.join(DIRNAME, 'misc', 'index.json')
+        f = join(DIRNAME, 'misc', 'index.json')
         bids_layout.load_index(f)
         assert bids_layout.unique('subject') == ['01']
         assert len(bids_layout.files) == 24
 
         # Test with reindexing
-        f = os.path.join(DIRNAME, 'misc', 'index.json')
+        f = join(DIRNAME, 'misc', 'index.json')
         bids_layout.load_index(f, reindex=True)
         assert bids_layout.unique('subject') == ['01']
         assert len(bids_layout.files) == 24
@@ -305,8 +328,8 @@ class TestLayout:
             def hash_file(self, file):
                 return str(hash(file.path)) + '.hsh'
 
-        root = os.path.join(DIRNAME, 'data', '7t_trt')
-        config = os.path.join(DIRNAME, 'specs',
+        root = join(DIRNAME, 'data', '7t_trt')
+        config = join(DIRNAME, 'specs',
                               'test_with_mapper.json')
 
         # Test with external mapper
@@ -337,10 +360,11 @@ class TestLayout:
 
     def test_excludes(self, tmpdir):
         root = tmpdir.mkdir("ohmyderivatives").mkdir("ds")
-        config = os.path.join(DIRNAME, 'specs', 'test.json')
+        config = join(DIRNAME, 'specs', 'test.json')
         layout = Layout(str(root), config, regex_search=True)
         assert layout._check_inclusions(str(root.join("ohmyimportantfile")))
-        assert not layout._check_inclusions(str(root.join("badbadderivatives")))
+        assert not layout._check_inclusions(
+            str(root.join("badbadderivatives")))
 
     def test_multiple_domains(self, stamp_layout):
         layout = stamp_layout.clone()