Repository: grabbles/grabbit

Issue:
Domain root should default to directory containing config file
It looks like if no root is specified in a directory-specific domain (or if `root == '.'`) then the root of that domain is set to the _global_ (`Layout`-level) root.  I'm confused by this behavior. I'd have intuitively thought that the domain's root would be the directory it's config file is taken from by default, and that relative paths would be relative to _that_ directory, rather than the layout root.  In fact, it seems to go against the purpose of the domains mechanisms to have to specify the relative path of the domain from the layout root in the domain-specific config file, when that information should be taken from the location of that file (by default).  Am I mis-understanding something?

As an example, adapted from the examples notebook:

```python
stamps = gb.Layout('../grabbit/tests/data/valuable_stamps/', absolute_paths=False, config_filename='dir_config.json', config='../grabbit/tests/specs/stamps.json')
[(k, d.root) for (k, d) in stamps.domains.items()]
```

gives 

```
[('stamps', '../grabbit/tests/data/valuable_stamps/'),
 ('usa_stamps', '../grabbit/tests/data/valuable_stamps/')]
```

while I would have expected

```
[('stamps', '../grabbit/tests/data/valuable_stamps/'),
 ('usa_stamps', '../grabbit/tests/data/valuable_stamps/USA/')]
```


Discussion:

Patch:
diff --git a/grabbit/core.py b/grabbit/core.py
index 166f17c..4a4eaa3 100644
--- a/grabbit/core.py
+++ b/grabbit/core.py
@@ -5,11 +5,13 @@ from collections import defaultdict, OrderedDict, namedtuple
 from grabbit.external import six, inflect
 from grabbit.utils import natural_sort, listify
 from grabbit.extensions.writable import build_path, write_contents_to_file
-from os.path import (join, basename, dirname, abspath, split, isabs, exists)
+from os.path import (join, basename, dirname, abspath, split, exists, isdir,
+                     relpath, isabs)
 from functools import partial
 from copy import deepcopy
 import warnings
 from keyword import iskeyword
+from itertools import chain
 
 
 __all__ = ['File', 'Entity', 'Layout']
@@ -66,12 +68,9 @@ class File(object):
 
             for name, val in entities.items():
 
-                if (name not in self.tags) ^ (val is None):
+                if name not in self.tags:
                     return False
 
-                if val is None:
-                    continue
-
                 def make_patt(x):
                     patt = '%s' % x
                     if isinstance(x, (int, float)):
@@ -121,11 +120,20 @@ class File(object):
         if new_filename[-1] == os.sep:
             new_filename += self.filename
 
+        if isabs(self.path) or root is None:
+            path = self.path
+        else:
+            path = join(root, self.path)
+
+        if not exists(path):
+            raise ValueError("Target filename to copy/symlink (%s) doesn't "
+                             "exist." % path)
+
         if symbolic_link:
             contents = None
-            link_to = self.path
+            link_to = path
         else:
-            with open(self.path, 'r') as f:
+            with open(path, 'r') as f:
                 contents = f.read()
             link_to = None
 
@@ -136,7 +144,7 @@ class File(object):
 
 class Domain(object):
 
-    def __init__(self, name, config, root):
+    def __init__(self, name, config):
         """
         A set of rules that applies to one or more directories
         within a Layout.
@@ -151,10 +159,8 @@ class Domain(object):
 
         self.name = name
         self.config = config
-        self.root = root
         self.entities = {}
         self.files = []
-        self.path_patterns = []
 
         self.include = listify(self.config.get('include', []))
         self.exclude = listify(self.config.get('exclude', []))
@@ -164,8 +170,7 @@ class Domain(object):
                              "both be set. Please pass at most one of these "
                              "for domain '%s'." % self.name)
 
-        if 'default_path_patterns' in config:
-            self.path_patterns += listify(config['default_path_patterns'])
+        self.path_patterns = listify(config.get('default_path_patterns', []))
 
     def add_entity(self, ent):
         ''' Add an Entity.
@@ -256,15 +261,14 @@ class Entity(object):
             setattr(result, k, new_val)
         return result
 
-    def matches(self, f, update_file=False):
+    def match_file(self, f, update_file=False):
         """
-        Determine whether the passed file matches the Entity and update the
-        Entity/File mappings.
+        Determine whether the passed file matches the Entity.
 
         Args:
             f (File): The File instance to match against.
-            update_file (bool): If True, the file's tag list is updated to
-                include the current Entity.
+
+        Returns: the matched value if a match was found, otherwise None.
         """
         if self.map_func is not None:
             val = self.map_func(f)
@@ -272,15 +276,10 @@ class Entity(object):
             m = self.regex.search(f.path)
             val = m.group(1) if m is not None else None
 
-        if val is None:
-            return False
+        if val is not None and self.dtype is not None:
+            val = self.dtype(val)
 
-        if update_file:
-            if self.dtype is not None:
-                val = self.dtype(val)
-            f.tags[self.name] = Tag(self, val)
-
-        return True
+        return val
 
     def add_file(self, filename, value):
         """ Adds the specified filename to tracking. """
@@ -320,21 +319,28 @@ class LayoutMetaclass(type):
 
 class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
-    def __init__(self, path, config=None, index=None, dynamic_getters=False,
-                 absolute_paths=True, regex_search=False, entity_mapper=None,
-                 path_patterns=None, config_filename='layout.json',
-                 include=None, exclude=None):
+    def __init__(self, root=None, config=None, index=None,
+                 dynamic_getters=False, absolute_paths=True,
+                 regex_search=False, entity_mapper=None, path_patterns=None,
+                 config_filename='layout.json', include=None, exclude=None):
         """
         A container for all the files and metadata found at the specified path.
 
         Args:
-            path (str): The root path of the layout.
-            config (str, list, dict): A specification of the configuration
-                file(s) defining domains to use in the Layout. Must be one of:
+            root (str): Directory that all other paths will be relative to.
+            Every other path the Layout sees must be at this level or below.
+            domains (str, list, dict): A specification of the configuration
+                object(s) defining domains to use in the Layout. Can be one of:
 
                 - A dictionary containing config information
                 - A string giving the path to a JSON file containing the config
-                - A list, where each element is one of the above
+                - A string giving the path to a directory containing a
+                  configuration file with the name defined in config_filename
+                - A tuple with two elements, where the first element is one of
+                  the above (i.e., dict or string), and the second element is
+                  an iterable of directories to apply the config to.
+                - A list, where each element is any of the above (dict, string,
+                  or tuple).
 
             index (str): Optional path to a saved index file. If a valid value
                 is passed, this index is used to populate Files and Entities,
@@ -386,7 +392,6 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
             raise ValueError("You cannot specify both the include and exclude"
                              " arguments. Please pass at most one of these.")
 
-        self.root = abspath(path) if absolute_paths else path
         self.entities = OrderedDict()
         self.files = {}
         self.mandatory = set()
@@ -398,91 +403,92 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         self.domains = OrderedDict()
         self.include = listify(include or [])
         self.exclude = listify(exclude or [])
+        self.absolute_paths = absolute_paths
+        self.root = abspath(root) if absolute_paths else root
 
         if config is not None:
             for c in listify(config):
-                self._load_domain(c)
+                if isinstance(c, tuple):
+                    c, root = c
+                else:
+                    root = None
+                self._load_domain(c, root, True)
 
         if index is None:
             self.index()
         else:
             self.load_index(index)
 
-    def _load_domain(self, config, root=None):
+    def _load_domain(self, config, root=None, from_init=False):
 
         if isinstance(config, six.string_types):
+
+            if isdir(config):
+                config = join(config, self.config_filename)
+
+            if not exists(config):
+                raise ValueError("Config file '%s' cannot be found." % config)
+
+            config_filename = config
             config = json.load(open(config, 'r'))
 
+            if root is None and not from_init:
+                root = dirname(abspath(config_filename))
+
         if 'name' not in config:
             raise ValueError("Config file missing 'name' attribute.")
+
         if config['name'] in self.domains:
             raise ValueError("Config with name '%s' already exists in "
                              "Layout. Name of each config file must be "
-                             "unique across entire Layout.")
-        if root is not None:
+                             "unique across entire Layout." % config['name'])
+
+        if root is None and from_init:
+            # warnings.warn("No valid root directory found for domain '%s'. "
+            #               "Falling back on root directory for Layout (%s)."
+            #               % (config['name'], self.root))
+            root = self.root
+
+        if config.get('root') in [None, '.']:
             config['root'] = root
 
-        if 'root' not in config:
-            warnings.warn("No valid root directory found for domain '%s'."
-                          " Falling back on the Layout's root directory. "
-                          "If this isn't the intended behavior, make sure "
-                          "the config file for this domain includes a "
-                          "'root' key." % config['name'])
-            config['root'] = self.root
-        elif config['root'] == '.':
-            config['root'] = self.root
-        elif not isabs(config['root']):
-            _root = config['root']
-            config['root'] = join(self.root, config['root'])
-            if not exists(config['root']):
-                msg = ("Relative path '%s' for domain '%s' interpreted as '%s'"
-                       ", but this directory doesn't exist. Either specify the"
-                       " domain root as an absolute path, or make sure it "
-                       "points to a valid directory when appended to the "
-                       "Layout's root (%s)." % (_root, config['name'],
-                                                config['root'], self.root))
-                raise ValueError(msg)
+        for root in listify(config['root']):
+            if not exists(root):
+                raise ValueError("Root directory %s for domain %s does not "
+                                 "exist!" % (root, config['name']))
 
         # Load entities
-        domain = Domain(config['name'], config, config['root'])
+        domain = Domain(config['name'], config)
         for e in config.get('entities', []):
             self.add_entity(domain=domain, **e)
 
         self.domains[domain.name] = domain
+        return domain
 
-    def get_domain_entities(self, domains=None, file=None):
+    def get_domain_entities(self, domains=None):
         # Get all Entities included in the specified Domains, in the same
-        # order as Domains in the list. Alternatively, if a file is passed,
-        # identify its domains and then return the entities.
-
-        if file is None:
-            if domains is None:
-                domains = list(self.domains.keys())
-        else:
-            domains = self._get_domains_for_file(file)
+        # order as Domains in the list.
+        if domains is None:
+            domains = list(self.domains.keys())
 
         ents = {}
         for d in domains:
             ents.update(self.domains[d].entities)
         return ents
 
-    def _check_inclusions(self, f, domains=None):
+    def _check_inclusions(self, f, domains=None, fullpath=True):
         ''' Check file or directory against regexes in config to determine if
             it should be included in the index '''
 
         filename = f if isinstance(f, six.string_types) else f.path
 
-        if os.path.isabs(filename) and filename.startswith(
-                self.root + os.path.sep):
-            # for filenames under the root - analyze relative path to avoid
-            # bringing injustice to the grandkids of some unfortunately named
-            # root directories.
-            filename = os.path.relpath(filename, self.root)
+        if not fullpath:
+            filename = basename(filename)
 
         if domains is None:
             domains = list(self.domains.keys())
 
-        domains = [self.domains[dom] for dom in domains]
+        domains = [self.domains[dom] for dom in listify(domains)]
 
         # Inject the Layout at the first position for global include/exclude
         domains.insert(0, self)
@@ -490,15 +496,14 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
             # If file matches any include regex, then True
             if dom.include:
                 for regex in dom.include:
-                    if re.match(regex, filename):
+                    if re.search(regex, filename):
                         return True
                 return False
             else:
                 # If file matches any exclude regex, then False
                 for regex in dom.exclude:
-                    if re.match(regex, filename, flags=re.UNICODE):
+                    if re.search(regex, filename, flags=re.UNICODE):
                         return False
-
         return True
 
     def _validate_dir(self, d):
@@ -515,10 +520,11 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         returned, the file will be ignored and dropped from the layout. '''
         return True
 
-    def _get_files(self):
+    def _get_files(self, root):
         ''' Returns all files in project (pre-filtering). Extend this in
         subclasses as needed. '''
-        return os.walk(self.root, topdown=True)
+        results = [os.walk(r, topdown=True) for r in listify(root)]
+        return list(chain(*results))
 
     def _make_file_object(self, root, f):
         ''' Initialize a new File oject from a directory and filename. Extend
@@ -531,51 +537,35 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         for ent in self.entities.values():
             ent.files = {}
 
-    def _get_domains_for_file(self, f):
-        if isinstance(f, File):
-            return f.domains
-        domains = []
-        for d in self.domains.values():
-            for path in listify(d.root):
-                if f.startswith(path):
-                    domains.append(d.name)
-                    break
-        return domains
-
-    def _index_file(self, root, f, domains=None, update_layout=True):
-
-        # If domains aren't explicitly passed, figure out what applies
-        if domains is None:
-            domains = self._get_domains_for_file(root)
+    def _index_file(self, root, f, domains, update_layout=True):
 
         # Create the file object--allows for subclassing
         f = self._make_file_object(root, f)
 
-        if not (self._check_inclusions(f, domains) and self._validate_file(f)):
-            return
-
-        for d in domains:
-            self.domains[d].add_file(f)
-
-        entities = self.get_domain_entities(domains)
-
-        if entities:
-            self.files[f.path] = f
+        for d in listify(domains):
+            if d not in self.domains:
+                raise ValueError("Cannot index file '%s' in domain '%s'; "
+                                 "no domain with that name exists." %
+                                 (f.path, d))
+            domain = self.domains[d]
+            match_vals = {}
+            for e in domain.entities.values():
+                m = e.match_file(f)
+                if m is None and e.mandatory:
+                    break
+                if m is not None:
+                    match_vals[e.name] = (e, m)
 
-        for e in entities.values():
-            e.matches(f, update_file=True)
+            if match_vals:
+                for k, (ent, val) in match_vals.items():
+                    f.tags[k] = Tag(ent, val)
+                    if update_layout:
+                        ent.add_file(f.path, val)
 
-        file_ents = f.tags.keys()
+            if update_layout:
+                domain.add_file(f)
 
-        # Only keep Files that match at least one Entity, and all
-        # mandatory Entities
-        if update_layout and file_ents and not (self.mandatory -
-                                                set(file_ents)):
-            self.files[f.path] = f
-            # Bind the File to all of the matching entities
-            for name, tag in f.tags.items():
-                ent_id = tag.entity.id
-                self.entities[ent_id].add_file(f.path, tag.value)
+        self.files[f.path] = f
 
         return f
 
@@ -598,50 +588,62 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
         self._reset_index()
 
-        dataset = self._get_files()
-
-        # Loop over all files
-        for root, directories, filenames in dataset:
-
-            # Determine which Domains apply to the current directory
-            domains = self._get_domains_for_file(root)
-
-            # Exclude directories that match exclude regex from further search
-            full_dirs = [os.path.join(root, d) for d in directories]
-
-            def check_incl(directory):
-                return self._check_inclusions(directory, domains)
-
-            full_dirs = filter(check_incl, full_dirs)
-            full_dirs = filter(self._validate_dir, full_dirs)
-            directories[:] = [split(d)[1] for d in full_dirs]
-
-            if self.config_filename in filenames:
-                config_path = os.path.join(root, self.config_filename)
-                config = json.load(open(config_path, 'r'))
-                cfg_root = config.get('root', root)
-                if cfg_root == '.':
-                    cfg_root = root
-                self._load_domain(config, root=cfg_root)
-
-                # Filter Domains if current dir's config file has an
-                # include directive
-                if 'domains' in config:
-                    missing = set(config['domains']) - set(domains)
-                    if missing:
-                        msg = ("Missing configs '%s' specified in include "
-                               "directive of config '%s'. Please make sure "
-                               "these config files are accessible from the "
-                               "directory %s.") % (missing, config['name'],
-                                                   root)
-                        raise ValueError(msg)
-                    domains = config['domains']
-                domains.append(config['name'])
-
-                filenames.remove(self.config_filename)
-
-            for f in filenames:
-                self._index_file(root, f, domains)
+        # Track all candidate files
+        files_to_index = defaultdict(set)
+
+        # Track any additional config files we run into
+        extra_configs = []
+
+        def _index_domain_files(dom):
+
+            doms_to_add = set(dom.config.get('domains', []) + [dom.name])
+
+            dataset = self._get_files(dom.config['root'])
+
+            # Loop over all files in domain
+            for root, directories, filenames in dataset:
+
+                def check_incl(f):
+                    return self._check_inclusions(f, dom.name)
+
+                # Exclude directories that match exclude regex
+                full_dirs = [join(root, d) for d in directories]
+                full_dirs = filter(check_incl, full_dirs)
+                full_dirs = filter(self._validate_dir, full_dirs)
+                directories[:] = [split(d)[1] for d in full_dirs]
+
+                for f in filenames:
+                    full_path = join(root, f)
+                    # Add config file to tracking
+                    if f == self.config_filename:
+                        if full_path not in extra_configs:
+                            extra_configs.append(full_path)
+                    # Add file to the candidate index
+                    elif (self._check_inclusions(full_path, dom.name) and
+                          self._validate_file(full_path)):
+                        # If the file is below the Layout root, use a relative
+                        # path. Otherwise, use absolute path.
+                        if full_path.startswith(self.root) and not \
+                                self.absolute_paths:
+                            full_path = relpath(full_path, self.root)
+                        files_to_index[full_path] |= doms_to_add
+
+        for dom in self.domains.values():
+            _index_domain_files(dom)
+
+        # Set up any additional configs we found. Note that in edge cases,
+        # this approach has the potential to miss out on some configs, because
+        # it doesn't recurse. This will generally only happen under fairly
+        # weird circumstances though (e.g., the config file points to another
+        # root elsewhere in the filesystem, or there are inconsistent include/
+        # exclude directives across nested configs), so this will do for now.
+        for dom in extra_configs:
+            dom = self._load_domain(dom)
+            _index_domain_files(dom)
+
+        for filename, domains in files_to_index.items():
+            _dir, _base = split(filename)
+            self._index_file(_dir, _base, list(domains))
 
     def save_index(self, filename):
         ''' Save the current Layout's index to a .json file.
@@ -656,7 +658,7 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         data = {}
         for f in self.files.values():
             entities = {v.entity.id: v.value for k, v in f.tags.items()}
-            data[f.path] = entities
+            data[f.path] = {'domains': f.domains, 'entities': entities}
         with open(filename, 'w') as outfile:
             json.dump(data, outfile)
 
@@ -679,15 +681,13 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
         self._reset_index()
         data = json.load(open(filename, 'r'))
 
-        for path, ents in data.items():
+        for path, file in data.items():
 
-            # If file path isn't absolute, assume it's relative to layout root
-            if not isabs(path):
-                path = join(self.root, path)
+            ents, domains = file['entities'], file['domains']
 
             root, f = dirname(path), basename(path)
             if reindex:
-                self._index_file(root, f)
+                self._index_file(root, f, domains)
             else:
                 f = self._make_file_object(root, f)
                 tags = {k: Tag(self.entities[k], v) for k, v in ents.items()}
@@ -717,8 +717,10 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
         if ent.mandatory:
             self.mandatory.add(ent.id)
+
         if ent.directory is not None:
             ent.directory = ent.directory.replace('{{root}}', self.root)
+
         self.entities[ent.id] = ent
         for alias in ent.aliases:
             self.entities[alias] = ent
@@ -1014,12 +1016,13 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
 
         for f in _files:
             f.copy(path_patterns, symbolic_link=symbolic_links,
-                   root=root, conflicts=conflicts)
+                   root=self.root, conflicts=conflicts)
 
     def write_contents_to_file(self, entities, path_patterns=None,
                                contents=None, link_to=None,
                                content_mode='text', conflicts='fail',
-                               strict=False, domains=None):
+                               strict=False, domains=None, index=False,
+                               index_domains=None):
         """
         Write arbitrary data to a file defined by the passed entities and
         path patterns.
@@ -1044,19 +1047,39 @@ class Layout(six.with_metaclass(LayoutMetaclass, object)):
             domains (list): List of Domains to scan for path_patterns. Order
                 determines precedence (i.e., earlier Domains will be scanned
                 first). If None, all available domains are included.
+            index (bool): If True, adds the generated file to the current
+                index using the domains specified in index_domains.
+            index_domains (list): List of domain names to attach the generated
+                file to when indexing. Ignored if index == False.  If None,
+                All available domains are used.
 
         """
-        if not path_patterns:
-            path_patterns = self.path_patterns
+        if path_patterns:
+            path = build_path(entities, path_patterns, strict)
+        else:
+            path_patterns = [self.path_patterns]
             if domains is None:
                 domains = list(self.domains.keys())
             for dom in domains:
-                path_patterns.extend(self.domains[dom].path_patterns)
-        path = build_path(entities, path_patterns, strict)
+                path_patterns.append(self.domains[dom].path_patterns)
+            for pp in path_patterns:
+                path = build_path(entities, pp, strict)
+                if path is not None:
+                    break
+
+        if path is None:
+            raise ValueError("Cannot construct any valid filename for "
+                             "the passed entities given available path "
+                             "patterns.")
+
         write_contents_to_file(path, contents=contents, link_to=link_to,
                                content_mode=content_mode, conflicts=conflicts,
                                root=self.root)
-        self._index_file(self.root, path)
+
+        if index:
+            if index_domains is None:
+                index_domains = list(self.domains.keys())
+            self._index_file(self.root, path, index_domains)
 
 
 def merge_layouts(layouts):


Test patch:
diff --git a/grabbit/tests/data/ordinary_stamps/name=1_Lotus#value=1#country=Canada.txt b/grabbit/tests/data/ordinary_stamps/name=1_Lotus#value=1#country=Canada.txt
new file mode 100644
index 0000000..e69de29
diff --git a/grabbit/tests/data/ordinary_stamps/name=50c_Forever_Stamp_2018#value=0.5#country=USA.txt b/grabbit/tests/data/ordinary_stamps/name=50c_Forever_Stamp_2018#value=0.5#country=USA.txt
new file mode 100644
index 0000000..e69de29
diff --git a/grabbit/tests/misc/index.json b/grabbit/tests/misc/index.json
index c3bbf1d..39f4f9b 100644
--- a/grabbit/tests/misc/index.json
+++ b/grabbit/tests/misc/index.json
@@ -1,130 +1,250 @@
 {
   "dataset_description.json": {
-    "test.type": "description"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "description"
+    }
   },
   "participants.tsv": {
-    "test.type": "trt/participants"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "trt/participants"
+    }
   },
   "task-rest_acq-fullbrain_bold.json": {
-    "test.type": "bold",
-    "test.task": "rest_acq"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "bold",
+      "test.task": "rest_acq"
+    }
   },
   "task-rest_acq-fullbrain_run-1_physio.json": {
-    "test.run": "1",
-    "test.type": "physio",
-    "test.task": "rest_acq",
-    "test.acquisition": "fullbrain_run"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.run": "1",
+      "test.type": "physio",
+      "test.task": "rest_acq",
+      "test.acquisition": "fullbrain_run"
+    }
   },
   "task-rest_acq-fullbrain_run-2_physio.json": {
-    "test.run": "2",
-    "test.type": "physio",
-    "test.task": "rest_acq",
-    "test.acquisition": "fullbrain_run"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.run": "2",
+      "test.type": "physio",
+      "test.task": "rest_acq",
+      "test.acquisition": "fullbrain_run"
+    }
   },
   "task-rest_acq-prefrontal_bold.json": {
-    "test.type": "bold",
-    "test.task": "rest_acq"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "bold",
+      "test.task": "rest_acq"
+    }
   },
   "task-rest_acq-prefrontal_physio.json": {
-    "test.type": "physio",
-    "test.task": "rest_acq"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "physio",
+      "test.task": "rest_acq"
+    }
   },
   "test.bval": {
-    "test.type": "trt/test",
-    "test.bval": "test.bval"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "trt/test",
+      "test.bval": "test.bval"
+    }
   },
   "models/excluded_model.json": {
-    "test.type": "model"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.type": "model"
+    }
   },
   "sub-01/sub-01_sessions.tsv": {
-    "test.subject": "01",
-    "test.type": "sessions"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.type": "sessions"
+    }
   },
   "sub-01/ses-1/sub-01_ses-1_scans.tsv": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.type": "scans"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.type": "scans"
+    }
   },
   "sub-01/ses-1/anat/sub-01_ses-1_T1map.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.type": "T1map"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.type": "T1map"
+    }
   },
   "sub-01/ses-1/anat/sub-01_ses-1_T1w.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.type": "T1w"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.type": "T1w"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-1_magnitude1.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "magnitude1"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "magnitude1"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-1_magnitude2.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "magnitude2"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "magnitude2"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-1_phasediff.json": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "phasediff"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "phasediff"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-1_phasediff.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "phasediff"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "phasediff"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-2_magnitude1.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "2",
-    "test.type": "magnitude1"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "2",
+      "test.type": "magnitude1"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-2_magnitude2.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "2",
-    "test.type": "magnitude2"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "2",
+      "test.type": "magnitude2"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-2_phasediff.json": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "2",
-    "test.type": "phasediff"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "2",
+      "test.type": "phasediff"
+    }
   },
   "sub-01/ses-1/fmap/sub-01_ses-1_run-2_phasediff.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "2",
-    "test.type": "phasediff"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "2",
+      "test.type": "phasediff"
+    }
   },
   "sub-01/ses-1/func/sub-01_ses-1_task-rest_acq-fullbrain_run-1_bold.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "bold",
-    "test.task": "rest_acq",
-    "test.acquisition": "fullbrain_run"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "bold",
+      "test.task": "rest_acq",
+      "test.acquisition": "fullbrain_run"
+    }
   },
   "sub-01/ses-1/func/sub-01_ses-1_task-rest_acq-fullbrain_run-1_physio.tsv.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "1",
-    "test.type": "physio",
-    "test.task": "rest_acq",
-    "test.acquisition": "fullbrain_run"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "1",
+      "test.type": "physio",
+      "test.task": "rest_acq",
+      "test.acquisition": "fullbrain_run"
+    }
   },
   "sub-01/ses-1/func/sub-01_ses-1_task-rest_acq-fullbrain_run-2_bold.nii.gz": {
-    "test.subject": "01",
-    "test.session": "1",
-    "test.run": "2",
-    "test.type": "bold",
-    "test.task": "rest_acq",
-    "test.acquisition": "fullbrain_run"
+    "domains": [
+      "test"
+    ],
+    "entities": {
+      "test.subject": "01",
+      "test.session": "1",
+      "test.run": "2",
+      "test.type": "bold",
+      "test.task": "rest_acq",
+      "test.acquisition": "fullbrain_run"
+    }
   }
-}
+}
\ No newline at end of file
diff --git a/grabbit/tests/specs/test.json b/grabbit/tests/specs/test.json
index 39ed11a..75ec8f4 100644
--- a/grabbit/tests/specs/test.json
+++ b/grabbit/tests/specs/test.json
@@ -5,14 +5,14 @@
     {
       "name": "subject",
       "pattern": "sub-(\\d+)",
-      "directory": "{{root}}/{subject}",
+      "directory": "{subject}",
       "dtype": "str"
     },
     {
       "name": "session",
       "pattern": "ses-0*(\\d+)",
       "mandatory": false,
-      "directory": "{{root}}/{subject}/{session}",
+      "directory": "{subject}/{session}",
       "missing_value": "ses-1"
     },
     {
diff --git a/grabbit/tests/test_core.py b/grabbit/tests/test_core.py
index d13da59..269140d 100644
--- a/grabbit/tests/test_core.py
+++ b/grabbit/tests/test_core.py
@@ -114,9 +114,8 @@ class TestEntity:
         tmpdir.mkdir("tmp").join(filename).write("###")
         f = File(join(str(tmpdir), filename))
         e = Entity('avaricious', 'aardvark-(\d+)')
-        e.matches(f, update_file=True)
-        assert 'avaricious' in f.entities
-        assert f.entities['avaricious'] == '4'
+        result = e.match_file(f)
+        assert result == '4'
 
     def test_unique_and_count(self):
         e = Entity('prop', '-(\d+)')
@@ -138,10 +137,6 @@ class TestEntity:
 class TestLayout:
 
     def test_init(self, bids_layout):
-        if hasattr(bids_layout, '_hdfs_client'):
-            assert bids_layout._hdfs_client.list(bids_layout.root)
-        else:
-            assert os.path.exists(bids_layout.root)
         assert isinstance(bids_layout.files, dict)
         assert isinstance(bids_layout.entities, dict)
         assert isinstance(bids_layout.mandatory, set)
@@ -169,26 +164,51 @@ class TestLayout:
         assert sub_file in bids_layout.files
         assert sub_file not in layout.files
 
+    def test_init_with_config_options(self):
+        root = join(DIRNAME, 'data')
+        dir1 = join(root, 'valuable_stamps')
+        dir2 = join(root, 'ordinary_stamps')
+        config1 = join(DIRNAME, 'specs', 'stamps.json')
+        config2 = join(dir1, 'USA', 'dir_config.json')
+
+        # Fails because Domain usa_stamps is included twice
+        with pytest.raises(ValueError) as e:
+            layout = Layout(root, [config1, config2], exclude=['7t_trt'],
+                            config_filename='dir_config.json')
+            assert e.value.message.startswith('Config with name')
+
+        # Test with two configs
+        layout = Layout(root, [config1, config2], exclude=['7t_trt'])
+        files = [f.filename for f in layout.files.values()]
+        assert 'name=Inverted_Jenny#value=75000#country=USA.txt' in files
+        assert 'name=5c_Francis_E_Willard#value=1dollar.txt' in files
+        assert 'name=1_Lotus#value=1#country=Canada.txt' in files
+
+        # Test with two configs and on-the-fly directory remapping
+        layout = Layout(dir1, [(config1, [dir1, dir2])],
+                        exclude=['USA/'])
+        files = [f.filename for f in layout.files.values()]
+        assert 'name=Inverted_Jenny#value=75000#country=USA.txt' in files
+        assert 'name=5c_Francis_E_Willard#value=1dollar.txt' not in files
+        assert 'name=1_Lotus#value=1#country=Canada.txt' in files
+
     def test_absolute_paths(self, bids_layout):
-        result = bids_layout.get(subject=1, run=1, session=1)
-        assert result  # that we got some entries
-        assert all([os.path.isabs(f.filename) for f in result])
 
         if not hasattr(bids_layout, '_hdfs_client'):
             root = join(DIRNAME, 'data', '7t_trt')
             root = os.path.relpath(root)
             config = join(DIRNAME, 'specs', 'test.json')
 
-            layout = Layout(root, config, absolute_paths=False)
+            layout = Layout(root, config, absolute_paths=True)
 
             result = layout.get(subject=1, run=1, session=1)
             assert result
-            assert not any([os.path.isabs(f.filename) for f in result])
+            assert all([os.path.isabs(f.filename) for f in result])
 
-            layout = Layout(root, config, absolute_paths=True)
+            layout = Layout(root, config, absolute_paths=False)
             result = layout.get(subject=1, run=1, session=1)
             assert result
-            assert all([os.path.isabs(f.filename) for f in result])
+            assert not any([os.path.isabs(f.filename) for f in result])
 
         # Should always be absolute paths on HDFS
         else:
@@ -250,8 +270,8 @@ class TestLayout:
         if hasattr(bids_layout, '_hdfs_client'):
             assert bids_layout._hdfs_client.list(bids_layout.root)
         else:
-            assert os.path.exists(result[0])
-            assert os.path.isdir(result[0])
+            assert os.path.exists(join(bids_layout.root, result[0]))
+            assert os.path.isdir(join(bids_layout.root, result[0]))
 
         result = bids_layout.get(target='subject', type='phasediff',
                                  return_type='file')
@@ -259,7 +279,8 @@ class TestLayout:
         if hasattr(bids_layout, '_hdfs_client'):
             assert all([bids_layout._hdfs_client.content(f) for f in result])
         else:
-            assert all([os.path.exists(f) for f in result])
+            assert all([os.path.exists(join(bids_layout.root, f))
+                        for f in result])
 
     def test_natsort(self, bids_layout):
         result = bids_layout.get(target='subject', return_type='id')
@@ -279,7 +300,7 @@ class TestLayout:
         nearest = bids_layout.get_nearest(
             result, type='sessions', extensions='tsv',
             ignore_strict_entities=['type'])
-        target = join('7t_trt', 'sub-01', 'sub-01_sessions.tsv')
+        target = join('sub-01', 'sub-01_sessions.tsv')
         assert target in nearest
         nearest = bids_layout.get_nearest(
             result, extensions='tsv', all_=True,
@@ -292,11 +313,11 @@ class TestLayout:
         assert nearest[0].subject == '01'
 
     def test_index_regex(self, bids_layout, layout_include):
-        targ = join(bids_layout.root, 'derivatives', 'excluded.json')
+        targ = join('derivatives', 'excluded.json')
         assert targ not in bids_layout.files
-        targ = join(layout_include.root, 'models',
-                            'excluded_model.json')
-        assert targ not in layout_include.files
+        domain = layout_include.domains['test_with_includes']
+        targ = join('models', 'excluded_model.json')
+        assert targ not in domain.files
 
         with pytest.raises(ValueError):
             layout_include._load_domain({'entities': [],
@@ -315,7 +336,8 @@ class TestLayout:
         for i in range(10):
             f = files[i]
             entities = {v.entity.id: v.value for v in f.tags.values()}
-            assert entities == index[f.path]
+            assert entities == index[f.path]['entities']
+            assert list(f.domains) == index[f.path]['domains']
         os.unlink(tmp)
 
     def test_load_index(self, bids_layout):
@@ -341,8 +363,7 @@ class TestLayout:
                 return str(hash(file.path)) + '.hsh'
 
         root = join(DIRNAME, 'data', '7t_trt')
-        config = join(DIRNAME, 'specs',
-                              'test_with_mapper.json')
+        config = join(DIRNAME, 'specs', 'test_with_mapper.json')
 
         # Test with external mapper
         em = EntityMapper()
@@ -363,7 +384,7 @@ class TestLayout:
 
     def test_clone(self, bids_layout):
         lc = bids_layout.clone()
-        attrs = ['root', 'mandatory', 'dynamic_getters', 'regex_search',
+        attrs = ['mandatory', 'dynamic_getters', 'regex_search',
                  'entity_mapper']
         for a in attrs:
             assert getattr(bids_layout, a) == getattr(lc, a)
@@ -374,7 +395,8 @@ class TestLayout:
         root = tmpdir.mkdir("ohmyderivatives").mkdir("ds")
         config = join(DIRNAME, 'specs', 'test.json')
         layout = Layout(str(root), config, regex_search=True)
-        assert layout._check_inclusions(str(root.join("ohmyimportantfile")))
+        assert layout._check_inclusions(str(root.join("ohmyimportantfile")),
+                                        fullpath=False)
         assert not layout._check_inclusions(
             str(root.join("badbadderivatives")))
 
@@ -383,6 +405,7 @@ class TestLayout:
         assert {'stamps', 'usa_stamps'} == set(layout.domains.keys())
         usa = layout.domains['usa_stamps']
         general = layout.domains['stamps']
+        print([f.filename for f in usa.files])
         assert len(usa.files) == 3
         assert len(layout.files) == len(general.files)
         assert not set(usa.files) - set(general.files)
diff --git a/grabbit/tests/test_writable.py b/grabbit/tests/test_writable.py
index 68b4442..5b1a8bf 100644
--- a/grabbit/tests/test_writable.py
+++ b/grabbit/tests/test_writable.py
@@ -18,7 +18,7 @@ def writable_file(tmpdir):
 def layout():
     data_dir = join(dirname(__file__), 'data', '7t_trt')
     config = join(dirname(__file__), 'specs', 'test.json')
-    layout = Layout(data_dir, config=config)
+    layout = Layout(data_dir, config, absolute_paths=False)
     return layout
 
 
@@ -176,8 +176,10 @@ class TestWritableLayout:
         data_dir = join(dirname(__file__), 'data', '7t_trt')
         entities = {'subject': 'Bob', 'session': '01'}
         pat = join('sub-{subject}/sess-{session}/desc.txt')
+
+        # With indexing
         layout.write_contents_to_file(entities, path_patterns=pat,
-                                      contents=contents)
+                                      contents=contents, index=True)
         target = join(data_dir, 'sub-Bob/sess-01/desc.txt')
         assert exists(target)
         with open(target) as f:
@@ -186,11 +188,23 @@ class TestWritableLayout:
         assert target in layout.files
         shutil.rmtree(join(data_dir, 'sub-Bob'))
 
+        # Without indexing
+        pat = join('sub-{subject}/sess-{session}/desc_no_index.txt')
+        layout.write_contents_to_file(entities, path_patterns=pat,
+                                      contents=contents, index=False)
+        target = join(data_dir, 'sub-Bob/sess-01/desc_no_index.txt')
+        assert exists(target)
+        with open(target) as f:
+            written = f.read()
+        assert written == contents
+        assert target not in layout.files
+        shutil.rmtree(join(data_dir, 'sub-Bob'))
+
     def test_write_contents_to_file_defaults(self, layout):
         contents = 'test'
         data_dir = join(dirname(__file__), 'data', '7t_trt')
         config = join(dirname(__file__), 'specs', 'test.json')
-        layout = Layout(data_dir, config=[config, {
+        layout = Layout(data_dir, [config, {
             'name': "test_writable",
             'default_path_patterns': ['sub-{subject}/ses-{session}/{subject}'
                                       '{session}{run}{type}{task}{acquisition}'
@@ -199,7 +213,7 @@ class TestWritableLayout:
         entities = {'subject': 'Bob', 'session': '01', 'run': '1',
                     'type': 'test', 'task': 'test', 'acquisition': 'test',
                     'bval': 0}
-        layout.write_contents_to_file(entities, contents=contents)
+        layout.write_contents_to_file(entities, contents=contents, index=True)
         target = join(data_dir, 'sub-Bob/ses-01/Bob011testtesttest0')
         assert exists(target)
         with open(target) as f:
@@ -218,6 +232,6 @@ class TestWritableLayout:
 
         data_dir = join(dirname(__file__), 'data', '7t_trt')
         filename = 'sub-04_ses-1_task-rest_acq-fullbrain_run-1_physio.tsv.gz'
-        file = join(data_dir, 'sub-04', 'ses-1', 'func', filename)
+        file = join('sub-04', 'ses-1', 'func', filename)
         path = layout.build_path(file, path_patterns=pat)
         assert path.endswith('sub-04/sess-1/r-1.nii.gz')