Repository: bids-standard/pybids

Issue:
memory issues when getting tsv in derivatives
Can't see this issue anywhere else. 

I can use `BIDSLaout.get()` to get the nifti files, and it does it in seconds. But if I try and get the confound tsv files instead, the python session will get killed cause it uses over 16GB of RAM to do it. 

> layout.get(scope='fMRIPrep', desc='confounds', suffix='regressors', extension='tsv', return_type='file')

Getting one subject will work use around 10GB of RAM (and getting per subject does not feel ideal). 

The following will work: 

> layout.get(scope='derivatives', desc='preproc', extension='nii.gz')

So the problem is only when getting tsv files. It doesn't matter if I set return_type to something else (I put return_type='file' in the example because its not just a problem when returning BIDSDataFile). There are not more confounds tsv files than preproc nifti files either. 

Is this a known/common problem or could there be a specific reason why I am hitting it? It feels counter-intuitive that getting the tsvs take up so much RAM. Is there a way around it which isn't just loop over subjects? 

## Information about dataset

Not sure if any of these details are relevant. But might be helpful.

I have a BIDS directory with ca 250 runs in total. There are about 27 subjects, 2 sessions and ca 5-9 runs for one session and 1-4 runs for another session. len(layout.files) is 8240. The derivatives are in fMRIPrep. There are some other directories in the derivatives, but nothing that is added. 

## Full code of what I am doing below: 

```
import bids
bids_dir = './' # pwd is in BIDS dataset
fmriprep_dir = 'derivatives/fmriprep-1.5.1/fmriprep/'
layout = bids.BIDSLayout(bids_dir)
layout.add_derivatives(bids_dir + fmriprep_dir)
# This will work
layout.get(scope='fMRIPrep', desc='preproc', extension='nii.gz')
# This will crash the session and use 100% memory on laptop with 16GB RAM
layout.get(scope='fMRIPrep', desc='confounds', suffix='regressors', extension='tsv', return_type='file')
```

Bids version: 0.9.4
Stochastic bug: sqlalchemy.orm.exc.DetachedInstanceError
Example: https://travis-ci.org/github/bids-standard/pybids/jobs/694363424#L828-L907

```
layout_7t_trt = BIDS Layout: .../pybids/bids/tests/data/7t_trt | Subjects: 10 | Sessions: 20 | Runs: 20
layout_7t_trt_relpath = BIDS Layout: .../pybids/bids/tests/data/7t_trt | Subjects: 10 | Sessions: 20 | Runs: 20
    def test_get_return_type_dir(layout_7t_trt, layout_7t_trt_relpath):
        query = dict(target='subject', return_type='dir')
        # In case of relative paths
>       res_relpath = layout_7t_trt_relpath.get(**query)
bids/layout/tests/test_layout.py:257:  
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
bids/layout/layout.py:1008: in get
    results = [x for x in results if target in x.entities]
bids/layout/layout.py:1008: in <listcomp>
    results = [x for x in results if target in x.entities]
venv/lib/python3.6/site-packages/sqlalchemy/ext/associationproxy.py:191: in __get__
    inst = self._as_instance(class_, obj)
venv/lib/python3.6/site-packages/sqlalchemy/ext/associationproxy.py:264: in _as_instance
    return inst._non_canonical_get_for_object(obj)
venv/lib/python3.6/site-packages/sqlalchemy/ext/associationproxy.py:792: in _non_canonical_get_for_object
    actual_obj = getattr(parent_instance, self.target_collection)
venv/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py:287: in __get__
    return self.impl.get(instance_state(instance), dict_)
venv/lib/python3.6/site-packages/sqlalchemy/orm/attributes.py:723: in get
    value = self.callable_(state, passive)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <sqlalchemy.orm.strategies.LazyLoader object at 0x7f5e76ae1990>
state = <sqlalchemy.orm.state.InstanceState object at 0x7f5e5a795ac8>
passive = symbol('PASSIVE_OFF')
    def _load_for_state(self, state, passive):

        if not state.key and (
            (
                not self.parent_property.load_on_pending
                and not state._load_pending
            )
            or not state.session_id
        ):
            return attributes.ATTR_EMPTY
    
        pending = not state.key
        primary_key_identity = None

        if (not passive & attributes.SQL_OK and not self.use_get) or (
            not passive & attributes.NON_PERSISTENT_OK and pending
        ):
            return attributes.PASSIVE_NO_RESULT
    
        if (
            # we were given lazy="raise"
            self._raise_always
            # the no_raise history-related flag was not passed
            and not passive & attributes.NO_RAISE
            and (
                # if we are use_get and related_object_ok is disabled,
                # which means we are at most looking in the identity map
                # for history purposes or otherwise returning
                # PASSIVE_NO_RESULT, don't raise.  This is also a
                # history-related flag
                not self.use_get
                or passive & attributes.RELATED_OBJECT_OK
            )
        ):

            self._invoke_raise_load(state, passive, "raise")

        session = _state_session(state)
        if not session:
            if passive & attributes.NO_RAISE:
                return attributes.PASSIVE_NO_RESULT

            raise orm_exc.DetachedInstanceError(
                "Parent instance %s is not bound to a Session; "
                "lazy load operation of attribute '%s' cannot proceed"
>               % (orm_util.state_str(state), self.key)
            )
E           sqlalchemy.orm.exc.DetachedInstanceError: Parent instance <BIDSImageFile at 0x7f5e833df150> is not bound to a Session; lazy load operation of attribute 'tags' cannot proceed (Background on this error at: http://sqlalche.me/e/bhk3)
venv/lib/python3.6/site-packages/sqlalchemy/orm/strategies.py:710: DetachedInstanceError
```

Discussion:

Patch:
diff --git a/bids/layout/__init__.py b/bids/layout/__init__.py
index 97dbff78..e93965d3 100644
--- a/bids/layout/__init__.py
+++ b/bids/layout/__init__.py
@@ -1,6 +1,7 @@
-from .layout import BIDSLayout, add_config_paths, parse_file_entities, Query
+from .layout import BIDSLayout, Query
 from .models import (BIDSFile, BIDSImageFile, BIDSDataFile, BIDSJSONFile,
                      Config, Entity, Tag)
+from .utils import add_config_paths, parse_file_entities
 # Backwards compatibility
 from bids_validator import BIDSValidator
 
diff --git a/bids/layout/db.py b/bids/layout/db.py
new file mode 100644
index 00000000..d3b4c06d
--- /dev/null
+++ b/bids/layout/db.py
@@ -0,0 +1,213 @@
+"""
+Database-related functionality.
+"""
+
+from pathlib import Path
+import json
+import re
+import warnings
+import sqlite3
+from functools import lru_cache
+
+import sqlalchemy as sa
+from sqlalchemy.orm import joinedload
+
+from bids.utils import listify
+from .models import Base, Config
+
+
+def get_database_file(path):
+    if path is not None:
+        path = Path(path)
+        database_file = path / 'layout_index.sqlite'
+        path.mkdir(exist_ok=True, parents=True)
+    else:
+        database_file = None
+    return database_file
+
+
+def get_database_sidecar(path):
+    """Given a path to a database file, return the associated sidecar.
+
+    Args:
+        path (str, Path): A path to a database file
+    """
+    if isinstance(path, str):
+        path = Path(path)
+    return path.parent / 'layout_args.json'
+
+
+def _sanitize_init_args(**kwargs):
+    """ Prepare initalization arguments for serialization """
+    # Make ignore and force_index serializable
+    for k in ['ignore', 'force_index']:
+        if kwargs.get(k) is not None:
+            kwargs[k] = [str(a) for a in kwargs.get(k) if a is not None]
+
+    if 'root' in kwargs:
+        kwargs['root'] = str(Path(kwargs['root']).absolute())
+
+    # Get abspaths
+    if 'derivatives' in kwargs and isinstance(kwargs['derivatives'], list):
+        kwargs['derivatives'] = [
+            str(Path(der).absolute())
+            for der in listify(kwargs['derivatives'])
+            ]
+
+    return kwargs
+
+
+class ConnectionManager:
+
+    def __init__(self, database_path=None, reset_database=False, config=None,
+                 init_args=None):
+
+        self.init_args = _sanitize_init_args(**(init_args or {}))
+        self.database_file = get_database_file(database_path)
+
+        # Determine if file exists before we create it in _get_engine()
+        reset_database = (
+            reset_database or                # manual reset
+            self.database_file is None or    # in-memory DB
+            not self.database_file.exists()  # file hasn't been created yet
+        )
+
+        self.engine = self._get_engine(self.database_file)
+        self.sessionmaker = sa.orm.sessionmaker(bind=self.engine)
+        self._session = None
+
+        if reset_database:
+            self.reset_database(config)
+        else:
+            self.load_database()
+
+        self._database_reset = reset_database
+
+    def _get_engine(self, database_file):
+        if database_file is not None:
+            # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html
+            # When a file-based database is specified, the dialect will use
+            # NullPool as the source of connections. This pool closes and
+            # discards connections which are returned to the pool immediately.
+            # SQLite file-based connections have extremely low overhead, so
+            # pooling is not necessary. The scheme also prevents a connection
+            # from being used again in a different thread and works best
+            # with SQLite's coarse-grained file locking.
+            from sqlalchemy.pool import NullPool
+            engine = sa.create_engine(
+                'sqlite:///{dbfilepath}'.format(dbfilepath=database_file),
+                connect_args={'check_same_thread': False},
+                poolclass=NullPool)
+        else:
+            # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html
+            # Using a Memory Database in Multiple Threads
+            # To use a :memory: database in a multithreaded scenario, the same
+            # connection object must be shared among threads, since the
+            # database exists only within the scope of that connection. The
+            # StaticPool implementation will maintain a single connection
+            # globally, and the check_same_thread flag can be passed to
+            # Pysqlite as False. Note that using a :memory: database in
+            # multiple threads requires a recent version of SQLite.
+            from sqlalchemy.pool import StaticPool
+            engine = sa.create_engine(
+                'sqlite://',  # In memory database
+                connect_args={'check_same_thread': False},
+                poolclass=StaticPool)
+            
+        def regexp(expr, item):
+            """Regex function for SQLite's REGEXP."""
+            reg = re.compile(expr, re.I)
+            return reg.search(item) is not None
+
+        engine.connect()
+
+        # Do not remove this decorator!!! An in-line create_function call will
+        # work when using an in-memory SQLite DB, but fails when using a file.
+        # For more details, see https://stackoverflow.com/questions/12461814/
+        @sa.event.listens_for(engine, "begin")
+        def do_begin(conn):
+            conn.connection.create_function('regexp', 2, regexp)
+
+        return engine
+
+    def reset_database(self, config=None):
+        Base.metadata.drop_all(self.engine)
+        Base.metadata.create_all(self.engine)
+        if self.database_sidecar is not None:
+            self.database_sidecar.write_text(json.dumps(self.init_args))
+        # Add config records
+        config = listify('bids' if config is None else config)
+        config = [Config.load(c, session=self.session) for c in listify(config)]
+        self.session.add_all(config)
+        self.session.commit()
+
+    def load_database(self):
+        if self.database_file is None:
+            raise ValueError("load_database() can only be called on databases "
+                             "stored in a file, not on in-memory databases.")
+        saved_args = json.loads(self.database_sidecar.read_text())
+        for k, v in saved_args.items():
+            curr_val = self.init_args.get(k)
+            if curr_val != v:
+                raise ValueError(
+                    "Initialization argument ('{}') does not match "
+                    "for database_path: {}.\n"
+                    "Saved value: {}.\n"
+                    "Current value: {}.".format(
+                        k, self.database_file, v, curr_val)
+                    )
+
+    def save_database(self, database_path, replace_connection=True):
+        """Save the current index as a SQLite3 DB at the specified location.
+
+        Note: This is only necessary if a database_path was not specified
+        at initialization, and the user now wants to save the index.
+        If a database_path was specified originally, there is no need to
+        re-save using this method.
+
+        Parameters
+        ----------
+        database_path : str
+            The path to the desired database folder. By default,
+            uses .db_cache. If a relative path is passed, it is assumed to
+            be relative to the BIDSLayout root directory.
+        replace_connection : bool, optional
+            If True, returns a new ConnectionManager that points to the newly
+            created database. If False, returns the current instance.
+        """
+        database_file = get_database_file(database_path)
+        database_sidecar = get_database_sidecar(database_file)
+        new_db = sqlite3.connect(str(database_file))
+        old_db = self.engine.connect().connection
+
+        with new_db:
+            for line in old_db.iterdump():
+                if line not in ('BEGIN;', 'COMMIT;'):
+                    new_db.execute(line)
+            new_db.commit()
+
+        # Dump instance arguments to JSON
+        database_sidecar.write_text(json.dumps(self.init_args))
+
+        if replace_connection:
+            return ConnectionManager(database_path, init_args=self.init_args)
+        else:
+            return self
+
+    @property
+    # Replace with @cached_property (3.8+) at some point in future
+    @lru_cache(maxsize=None)
+    def database_sidecar(self):
+        if self.database_file is not None:
+            return get_database_sidecar(self.database_file)
+        return None
+
+    @property
+    def session(self):
+        if self._session is None:
+            self.reset_session()
+        return self._session
+
+    def reset_session(self):
+        """Force a new session."""
+        self._session = self.sessionmaker()
\ No newline at end of file
diff --git a/bids/layout/index.py b/bids/layout/index.py
index f533af9b..fef38a6d 100644
--- a/bids/layout/index.py
+++ b/bids/layout/index.py
@@ -46,13 +46,13 @@ class BIDSLayoutIndexer(object):
     def __init__(self, layout):
 
         self.layout = layout
+        self.config = list(layout.config.values())
         self.session = layout.session
         self.validate = layout.validate
         self.root = layout.root
         self.config_filename = layout.config_filename
         self.validator = BIDSValidator(index_associated=True)
         # Create copies of list attributes we'll modify during indexing
-        self.config = list(layout.config.values())
         self.include_patterns = list(layout.force_index)
         self.exclude_patterns = list(layout.ignore)
 
@@ -164,11 +164,11 @@ class BIDSLayoutIndexer(object):
 
         return bf
 
-    def index_files(self):
+    def add_files(self):
         """Index all files in the BIDS dataset. """
         self._index_dir(self.root, self.config)
 
-    def index_metadata(self, **filters):
+    def add_metadata(self, **filters):
         """Index metadata for all files in the BIDS dataset.
 
         Parameters
diff --git a/bids/layout/layout.py b/bids/layout/layout.py
index 56689b9d..18840ed1 100644
--- a/bids/layout/layout.py
+++ b/bids/layout/layout.py
@@ -8,31 +8,28 @@ from functools import partial
 from itertools import chain
 import copy
 import warnings
-import sqlite3
 import enum
-from pathlib import Path
 import difflib
 
 import sqlalchemy as sa
-from sqlalchemy.orm import joinedload
 from bids_validator import BIDSValidator
 
-from ..utils import listify, natural_sort, make_bidsfile
+from ..utils import listify, natural_sort
 from ..external import inflect
-from .writing import build_path, write_to_file
-from .models import (Base, Config, BIDSFile, Entity, Tag)
-from .index import BIDSLayoutIndexer
-from .utils import BIDSMetadata
-from .. import config as cf
 from ..exceptions import (
     BIDSDerivativesValidationError,
     BIDSEntityError,
     BIDSValidationError,
-    ConfigError,
     NoMatchError,
     TargetError,
 )
 
+from .writing import build_path, write_to_file
+from .models import (Base, Config, BIDSFile, Entity, Tag)
+from .index import BIDSLayoutIndexer
+from .db import ConnectionManager, get_database_sidecar, get_database_file
+from .utils import (BIDSMetadata, parse_file_entities)
+
 try:
     from os.path import commonpath
 except ImportError:
@@ -61,84 +58,6 @@ EXAMPLE_DERIVATIVES_DESCRIPTION = {
     k: val[k] for val in MANDATORY_DERIVATIVES_FIELDS.values() for k in val}
 
 
-def parse_file_entities(filename, entities=None, config=None,
-                        include_unmatched=False):
-    """Parse the passed filename for entity/value pairs.
-
-    Parameters
-    ----------
-    filename : str
-        The filename to parse for entity values
-    entities : list or None, optional
-        An optional list of Entity instances to use in extraction.
-        If passed, the config argument is ignored. Default is None.
-    config : str or :obj:`bids.layout.models.Config` or list or None, optional
-        One or more :obj:`bids.layout.models.Config` objects or names of
-        configurations to use in matching. Each element must be a
-        :obj:`bids.layout.models.Config` object, or a valid
-        :obj:`bids.layout.models.Config` name (e.g., 'bids' or 'derivatives').
-        If None, all available configs are used. Default is None.
-    include_unmatched : bool, optional
-        If True, unmatched entities are included in the returned dict,
-        with values set to None.
-        If False (default), unmatched entities are ignored.
-
-    Returns
-    -------
-    dict
-        Keys are Entity names and values are the values from the filename.
-    """
-    # Load Configs if needed
-    if entities is None:
-
-        if config is None:
-            config = ['bids', 'derivatives']
-
-        config = [Config.load(c) if not isinstance(c, Config) else c
-                  for c in listify(config)]
-
-        # Consolidate entities from all Configs into a single dict
-        entities = {}
-        for c in config:
-            entities.update(c.entities)
-        entities = entities.values()
-
-    # Extract matches
-    bf = make_bidsfile(filename)
-    ent_vals = {}
-    for ent in entities:
-        match = ent.match_file(bf)
-        if match is not None or include_unmatched:
-            ent_vals[ent.name] = match
-
-    return ent_vals
-
-
-def add_config_paths(**kwargs):
-    """Add to the pool of available configuration files for BIDSLayout.
-
-    Parameters
-    ----------
-    kwargs : dict
-        Dictionary specifying where to find additional config files.
-        Keys are names, values are paths to the corresponding .json file.
-
-    Examples
-    --------
-    > add_config_paths(my_config='/path/to/config')
-    > layout = BIDSLayout('/path/to/bids', config=['bids', 'my_config'])
-    """
-    for k, path in kwargs.items():
-        if not os.path.exists(path):
-            raise ConfigError(
-                'Configuration file "{}" does not exist'.format(k))
-        if k in cf.get_option('config_paths'):
-            raise ConfigError('Configuration {!r} already exists'.format(k))
-
-    kwargs.update(**cf.get_option('config_paths'))
-    cf.set_option('config_paths', kwargs)
-
-
 class BIDSLayout(object):
     """Layout class representing an entire BIDS dataset.
 
@@ -224,7 +143,7 @@ class BIDSLayout(object):
                  force_index=None, config_filename='layout_config.json',
                  regex_search=False, database_path=None, database_file=None,
                  reset_database=False, index_metadata=True):
-        """Initialize BIDSLayout."""
+
         self.root = root
         self.validate = validate
         self.absolute_paths = absolute_paths
@@ -232,32 +151,21 @@ class BIDSLayout(object):
         self.sources = sources
         self.regex_search = regex_search
         self.config_filename = config_filename
-        # Store original init arguments as dictionary
-        self._init_args = self._sanitize_init_args(
-            root=root, validate=validate, absolute_paths=absolute_paths,
-            derivatives=derivatives, ignore=ignore, force_index=force_index,
-            index_metadata=index_metadata, config=config)
+
+        if ignore is None:
+            ignore = self._default_ignore
 
         if database_path is None and database_file is not None:
             database_path = database_file
             warnings.warn(
                 'In pybids 0.10 database_file argument was deprecated in favor'
                 ' of database_path, and will be removed in 0.12. '
-                'For now, treating database_file as a directory.',
+                'For now, interpreting database_file as a directory.',
                 DeprecationWarning)
-        if database_path:
-            database_path = str(Path(database_path).absolute())
-
-        self.session = None
-
-        index_dataset = self._init_db(database_path, reset_database)
 
         # Do basic BIDS validation on root directory
         self._validate_root()
 
-        if ignore is None:
-            ignore = self._default_ignore
-
         # Instantiate after root validation to ensure os.path.join works
         self.ignore = [os.path.abspath(os.path.join(self.root, patt))
                        if isinstance(patt, str) else patt
@@ -269,26 +177,23 @@ class BIDSLayout(object):
         # Initialize the BIDS validator and examine ignore/force_index args
         self._validate_force_index()
 
-        if index_dataset:
-            # Create Config objects
-            if config is None:
-                config = 'bids'
-            config = [Config.load(c, session=self.session)
-                      for c in listify(config)]
-            self.config = {c.name: c for c in config}
-            # Missing persistence of configs to the database
-            for config_obj in self.config.values():
-                self.session.add(config_obj)
-                self.session.commit()
-
-            # Index files and (optionally) metadata
-            indexer = BIDSLayoutIndexer(self)
-            indexer.index_files()
+        init_args = dict(
+            root=root, validate=validate, absolute_paths=absolute_paths,
+            derivatives=derivatives, ignore=ignore, force_index=force_index,
+            index_metadata=index_metadata, config=config)
+
+        # Set up the DB
+        self.connection_manager = ConnectionManager(
+            database_path, reset_database, config, init_args)
+
+        self.config = {c.name: c for c in self.session.query(Config).all()}
+
+        # Index project if needed
+        self.indexer = BIDSLayoutIndexer(self)
+        if self.connection_manager._database_reset:
+            self.indexer.add_files()
             if index_metadata:
-                indexer.index_metadata()
-        else:
-            # Load Configs from DB
-            self.config = {c.name: c for c in self.session.query(Config).all()}
+                self.indexer.add_metadata()
 
         # Add derivatives if any are found
         if derivatives:
@@ -300,7 +205,8 @@ class BIDSLayout(object):
                 derivatives=None, sources=self, ignore=ignore,  config=None,
                 force_index=force_index, config_filename=config_filename,
                 regex_search=regex_search, index_metadata=index_metadata,
-                reset_database=index_dataset or reset_database
+                # reset_database=index_dataset or reset_database
+                reset_database=reset_database
                 )
 
     def __getattr__(self, key):
@@ -354,121 +260,6 @@ class BIDSLayout(object):
              "Runs: {}".format(root, n_subjects, n_sessions, n_runs))
         return s
 
-    def _set_session(self, database_file):
-        if database_file is not None:
-            # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html
-            # When a file-based database is specified, the dialect will use
-            # NullPool as the source of connections. This pool closes and
-            # discards connections which are returned to the pool immediately.
-            # SQLite file-based connections have extremely low overhead, so
-            # pooling is not necessary. The scheme also prevents a connection
-            # from being used again in a different thread and works best
-            # with SQLite's coarse-grained file locking.
-            from sqlalchemy.pool import NullPool
-            engine = sa.create_engine(
-                'sqlite:///{dbfilepath}'.format(dbfilepath=database_file),
-                connect_args={'check_same_thread': False},
-                poolclass=NullPool)
-        else:
-            # https://docs.sqlalchemy.org/en/13/dialects/sqlite.html
-            # Using a Memory Database in Multiple Threads
-            # To use a :memory: database in a multithreaded scenario, the same
-            # connection object must be shared among
-            # threads, since the database exists only within the scope of that
-            # connection. The StaticPool implementation will maintain a single
-            # connection globally, and the check_same_thread flag can be passed
-            # to Pysqlite as False:
-            from sqlalchemy.pool import StaticPool
-            engine = sa.create_engine(
-                'sqlite://',  # In memory database
-                connect_args={'check_same_thread': False},
-                poolclass=StaticPool)
-            # Note that using a :memory: database in multiple threads requires
-            # a recent version of SQLite.
-
-        def regexp(expr, item):
-            """Regex function for SQLite's REGEXP."""
-            reg = re.compile(expr, re.I)
-            return reg.search(item) is not None
-
-        conn = engine.connect()
-
-        # Do not remove this decorator!!! An in-line create_function call will
-        # work when using an in-memory SQLite DB, but fails when using a file.
-        # For more details, see https://stackoverflow.com/questions/12461814/
-        @sa.event.listens_for(engine, "begin")
-        def do_begin(conn):
-            conn.connection.create_function('regexp', 2, regexp)
-
-        self.session = sa.orm.sessionmaker(bind=engine)()
-
-    @staticmethod
-    def _make_db_paths(database_path):
-        if database_path is not None:
-            database_path = Path(database_path)
-            database_file = database_path / 'layout_index.sqlite'
-            database_sidecar = database_path / 'layout_args.json'
-            database_path.mkdir(exist_ok=True, parents=True)
-        else:
-            database_file = None
-            database_sidecar = None
-        return database_file, database_sidecar
-
-    @staticmethod
-    def _sanitize_init_args(**kwargs):
-        """ Prepare initalization arguments for serialization """
-        # Make ignore and force_index serializable
-        for k in ['ignore', 'force_index']:
-            kwargs[k] = [
-                str(a) for a in kwargs[k] if a is not None] \
-              if kwargs[k] is not None else None
-
-        kwargs['root'] = str(Path(kwargs['root']).absolute())
-
-        # Get abspaths
-        if isinstance(kwargs['derivatives'], list):
-            kwargs['derivatives'] = [
-                str(Path(der).absolute())
-                for der in listify(kwargs['derivatives'])
-                ]
-
-        return kwargs
-
-    def _init_db(self, database_path=None, reset_database=False):
-        database_file, database_sidecar = self._make_db_paths(database_path)
-        # Reset database if needed and return whether or not it was reset.
-        # Determining if the database needs resetting must be done prior
-        # to setting the session (which creates the empty database file)
-        reset_database = (
-            reset_database or  # Manual Request
-            not database_file or  # In memory transient db
-            not database_file.exists()  # New file based db created
-        )
-
-        self._set_session(database_file)
-
-        if not reset_database:
-            saved_args = json.loads(database_sidecar.read_text())
-            for k, v in saved_args.items():
-                if self._init_args[k] != v:
-                    raise ValueError(
-                        "Initialization argument ('{}') does not match "
-                        "for database_path: {}.\n"
-                        "Saved value: {}.\n"
-                        "Current value: {}.".format(
-                            k, database_path, v, self._init_args[k])
-                        )
-        else:
-            engine = self.session.get_bind()
-            Base.metadata.drop_all(engine)
-            Base.metadata.create_all(engine)
-            if database_sidecar:
-                database_sidecar.write_text(json.dumps(self._init_args))
-
-            return True
-
-        return False
-
     def _validate_root(self):
         # Validate root argument and make sure it contains mandatory info
 
@@ -577,6 +368,10 @@ class BIDSLayout(object):
                 pass
         return entities
 
+    @property
+    def session(self):
+        return self.connection_manager.session
+
     @property
     def entities(self):
         """Get the entities."""
@@ -594,14 +389,14 @@ class BIDSLayout(object):
 
         Parameters
         ----------
-        database_path : str
-            The path to the desired database folder. By default,
-            uses .db_cache. If a relative path is passed, it is assumed to
-            be relative to the BIDSLayout root directory.
+        database_path : str, Path
+            The path to the desired database folder. If a relative path is
+            passed, it is assumed to be relative to the BIDSLayout root
+            directory.
         """
-        database_file, database_sidecar = cls._make_db_paths(database_path)
+        database_file = get_database_file(database_path)
+        database_sidecar = get_database_sidecar(database_file)
         init_args = json.loads(database_sidecar.read_text())
-
         return cls(database_path=database_path, **init_args)
 
     def save(self, database_path, replace_connection=True):
@@ -627,27 +422,15 @@ class BIDSLayout(object):
             be reflected in the new file unless save() is explicitly called
             again.
         """
-        database_file, database_sidecar = self._make_db_paths(database_path)
-        new_db = sqlite3.connect(str(database_file))
-        old_db = self.session.get_bind().connect().connection
-
-        with new_db:
-            for line in old_db.iterdump():
-                if line not in ('BEGIN;', 'COMMIT;'):
-                    new_db.execute(line)
-            new_db.commit()
-
-        if replace_connection:
-            self._set_session(str(database_file))
-
-        # Dump instance arguments to JSON
-        database_sidecar.write_text(json.dumps(self._init_args))
+        self.connection_manager = self.connection_manager.save_database(
+            database_path, replace_connection)
 
         # Recursively save children
         for pipeline_name, der in self.derivatives.items():
             der.save(os.path.join(
                 database_path, pipeline_name))
 
+
     def get_entities(self, scope='all', metadata=None):
         """Get entities for all layouts in the specified scope.
 
@@ -999,10 +782,14 @@ class BIDSLayout(object):
         for l in layouts:
             query = l._build_file_query(filters=filters,
                                         regex_search=regex_search)
-            # Eager load associations, because mixing queries from different
-            # DB sessions causes objects to detach
-            query = query.options(joinedload(BIDSFile.tags)
-                                  .joinedload(Tag.entity))
+            # NOTE: The following line, when uncommented, eager loads
+            # associations. This was introduced in order to prevent sessions
+            # from randomly detaching. It should be fixed by setting
+            # expire_on_commit at session creation, but let's leave this here
+            # for another release or two to make sure we don't have any further
+            # problems.
+            # query = query.options(joinedload(BIDSFile.tags)
+            #                       .joinedload(Tag.entity))
             results.extend(query.all())
 
         # Convert to relative paths if needed
@@ -1576,7 +1363,7 @@ class BIDSLayout(object):
             Whether to copy each file as a symbolic link or a deep copy.
         root : str
             Optional root directory that all patterns are relative
-            to. Defaults to current working directory.
+            to. Defaults to dataset root.
         conflicts : str
             Defines the desired action when the output path already exists.
             Must be one of:
@@ -1587,13 +1374,15 @@ class BIDSLayout(object):
         kwargs : dict
             Optional key word arguments to pass into a get() query.
         """
+        root = self.root if root is None else root
+
         _files = self.get(**kwargs)
         if files:
             _files = list(set(files).intersection(_files))
 
         for f in _files:
             f.copy(path_patterns, symbolic_link=symbolic_links,
-                   root=self.root, conflicts=conflicts)
+                   root=root, conflicts=conflicts)
 
     def write_to_file(self, entities, path_patterns=None,
                       contents=None, link_to=None, copy_from=None,
diff --git a/bids/layout/utils.py b/bids/layout/utils.py
index 847b7408..33cc328c 100644
--- a/bids/layout/utils.py
+++ b/bids/layout/utils.py
@@ -1,3 +1,12 @@
+"""Miscellaneous layout-related utilities."""
+import os
+from pathlib import Path
+
+from .. import config as cf
+from ..utils import make_bidsfile, listify
+from ..exceptions import ConfigError
+
+
 class BIDSMetadata(dict):
     """ Metadata dictionary that reports the associated file on lookup failures. """
     def __init__(self, source_file):
@@ -10,3 +19,82 @@ class BIDSMetadata(dict):
         except KeyError as e:
             raise KeyError(
                 "Metadata term {!r} unavailable for file {}.".format(key, self._source_file))
+
+
+def parse_file_entities(filename, entities=None, config=None,
+                        include_unmatched=False):
+    """Parse the passed filename for entity/value pairs.
+
+    Parameters
+    ----------
+    filename : str
+        The filename to parse for entity values
+    entities : list or None, optional
+        An optional list of Entity instances to use in extraction.
+        If passed, the config argument is ignored. Default is None.
+    config : str or :obj:`bids.layout.models.Config` or list or None, optional
+        One or more :obj:`bids.layout.models.Config` objects or names of
+        configurations to use in matching. Each element must be a
+        :obj:`bids.layout.models.Config` object, or a valid
+        :obj:`bids.layout.models.Config` name (e.g., 'bids' or 'derivatives').
+        If None, all available configs are used. Default is None.
+    include_unmatched : bool, optional
+        If True, unmatched entities are included in the returned dict,
+        with values set to None.
+        If False (default), unmatched entities are ignored.
+
+    Returns
+    -------
+    dict
+        Keys are Entity names and values are the values from the filename.
+    """
+    # Load Configs if needed
+    if entities is None:
+
+        if config is None:
+            config = ['bids', 'derivatives']
+
+        from .models import Config
+        config = [Config.load(c) if not isinstance(c, Config) else c
+                  for c in listify(config)]
+
+        # Consolidate entities from all Configs into a single dict
+        entities = {}
+        for c in config:
+            entities.update(c.entities)
+        entities = entities.values()
+
+    # Extract matches
+    bf = make_bidsfile(filename)
+    ent_vals = {}
+    for ent in entities:
+        match = ent.match_file(bf)
+        if match is not None or include_unmatched:
+            ent_vals[ent.name] = match
+
+    return ent_vals
+
+
+def add_config_paths(**kwargs):
+    """Add to the pool of available configuration files for BIDSLayout.
+
+    Parameters
+    ----------
+    kwargs : dict
+        Dictionary specifying where to find additional config files.
+        Keys are names, values are paths to the corresponding .json file.
+
+    Examples
+    --------
+    > add_config_paths(my_config='/path/to/config')
+    > layout = BIDSLayout('/path/to/bids', config=['bids', 'my_config'])
+    """
+    for k, path in kwargs.items():
+        if not os.path.exists(path):
+            raise ConfigError(
+                'Configuration file "{}" does not exist'.format(k))
+        if k in cf.get_option('config_paths'):
+            raise ConfigError('Configuration {!r} already exists'.format(k))
+
+    kwargs.update(**cf.get_option('config_paths'))
+    cf.set_option('config_paths', kwargs)


Test patch:
diff --git a/bids/layout/tests/conftest.py b/bids/layout/tests/conftest.py
index 720d8241..5920edf9 100644
--- a/bids/layout/tests/conftest.py
+++ b/bids/layout/tests/conftest.py
@@ -1,7 +1,7 @@
 from os.path import join
 
 import pytest
-import tempfile
+
 from bids.layout import BIDSLayout
 from bids.tests import get_test_data_path
 
diff --git a/bids/layout/tests/test_db.py b/bids/layout/tests/test_db.py
new file mode 100644
index 00000000..f844d96f
--- /dev/null
+++ b/bids/layout/tests/test_db.py
@@ -0,0 +1,37 @@
+"""Test functionality in the db module--mostly related to connection
+management."""
+
+import re
+from pathlib import Path
+
+from bids.layout.db import (ConnectionManager, get_database_file,
+                            get_database_sidecar, _sanitize_init_args)
+
+
+def test_sanitize_init_args():
+    patt = [re.compile('f1ct.*n'), 'code']
+    root = Path('.') / 'fictional_path'
+    result = _sanitize_init_args(ignore=patt, root=root, absolute_paths=False)
+    assert result['absolute_paths'] == False
+    assert isinstance(result['ignore'], list)
+    assert all([isinstance(el, str) for el in result['ignore']])
+    assert isinstance(result['root'], str)
+
+
+def test_get_database_file(tmp_path):
+    tmp_path = Path(str(tmp_path))  # PY35: pytest uses pathlib2.Path; __fspath__ in Python 3.6 fixes this
+    assert get_database_file(None) is None
+    new_path = tmp_path / "a_new_subdir"
+    assert not new_path.exists()
+    db_file = get_database_file(new_path)
+    assert db_file == new_path / 'layout_index.sqlite'
+    assert new_path.exists()
+
+
+def test_get_database_sidecar():
+    db_file = '/abs/path/to/db/file.sqlite'
+    f1 = get_database_sidecar(db_file)
+    assert isinstance(f1, Path)
+    assert str(f1) == '/abs/path/to/db/layout_args.json'
+    f2 = get_database_sidecar(Path(db_file))
+    assert f1 == f2
diff --git a/bids/layout/tests/test_layout.py b/bids/layout/tests/test_layout.py
index b98e3c47..d33424df 100644
--- a/bids/layout/tests/test_layout.py
+++ b/bids/layout/tests/test_layout.py
@@ -1,26 +1,21 @@
-""" Tests of BIDS-specific functionality. Generic tests of core grabbit
-functionality should go in the grabbit package. """
+""" Tests of functionality in the layout module--mostly related to the
+BIDSLayout class."""
 
 import os
 import re
-import tempfile
-from os.path import join, abspath, basename, dirname
+from os.path import (join, abspath, basename)
 
 import numpy as np
 import pytest
 
-import bids
-from bids.layout import (BIDSLayout, parse_file_entities, add_config_paths,
-                         Query)
+from bids.layout import BIDSLayout, Query
 from bids.layout.index import BIDSLayoutIndexer
-from bids.layout.models import Entity, Config
-from bids.layout.utils import BIDSMetadata
+from bids.layout.models import Config
 from bids.tests import get_test_data_path
 from bids.utils import natural_sort
 
 from bids.exceptions import (
     BIDSValidationError,
-    ConfigError,
     NoMatchError,
     TargetError,
 )
@@ -47,7 +42,7 @@ def test_index_metadata(index_metadata, query, result, mock_config):
     layout = BIDSLayout(data_dir, index_metadata=index_metadata)
     if not index_metadata and query is not None:
         indexer = BIDSLayoutIndexer(layout)
-        indexer.index_metadata(**query)
+        indexer.add_metadata(**query)
     sample_file = layout.get(task='rest', extension='.nii.gz',
                              acquisition='fullbrain')[0]
     metadata = sample_file.get_metadata()
@@ -274,6 +269,17 @@ def test_get_return_type_dir(layout_7t_trt, layout_7t_trt_relpath):
     assert target == res2
 
 
+def test_ensure_non_detaching_sessions(layout_7t_trt, layout_7t_trt_relpath):
+    """ Ensure that sessions don't detach unexpectly during queries."""
+    query = dict(target='subject', return_type='dir')
+    res_relpath = layout_7t_trt_relpath.get(**query)
+    target_relpath = ["sub-{:02d}".format(i) for i in range(1, 11)]
+    assert target_relpath == res_relpath
+    res_abspath = layout_7t_trt.get(**query)
+    # Second query would break without expire_on_commit=True
+    res_relpath = layout_7t_trt_relpath.get(**query)
+
+
 @pytest.mark.parametrize("acq", [None, Query.NONE])
 def test_get_val_none(layout_7t_trt, acq):
     t1w_files = layout_7t_trt.get(subject='01', session='1', suffix='T1w')
@@ -478,38 +484,6 @@ def test_to_df(layout_ds117):
                 set(df.columns))
 
 
-@pytest.mark.parametrize("extension_initial_dot", (True, False))
-def test_parse_file_entities(mock_config, extension_initial_dot):
-    filename = '/sub-03_ses-07_run-4_desc-bleargh_sekret.nii.gz'
-
-    dot = '.' if extension_initial_dot else ''
-
-    # Test with entities taken from bids config
-    target = {'subject': '03', 'session': '07', 'run': 4, 'suffix': 'sekret',
-              'extension': dot + 'nii.gz'}
-    assert target == parse_file_entities(filename, config='bids')
-    config = Config.load('bids')
-    assert target == parse_file_entities(filename, config=[config])
-
-    # Test with entities taken from bids and derivatives config
-    target = {'subject': '03', 'session': '07', 'run': 4, 'suffix': 'sekret',
-              'desc': 'bleargh', 'extension': dot + 'nii.gz'}
-    assert target == parse_file_entities(filename)
-    assert target == parse_file_entities(
-        filename, config=['bids', 'derivatives'])
-
-    # Test with list of Entities
-    entities = [
-        Entity('subject', "[/\\\\]sub-([a-zA-Z0-9]+)"),
-        Entity('run', "[_/\\\\]run-0*(\\d+)", dtype=int),
-        Entity('suffix', "[._]*([a-zA-Z0-9]*?)\\.[^/\\\\]+$"),
-        Entity('desc', "desc-([a-zA-Z0-9]+)"),
-    ]
-    # Leave out session to distinguish from previous test target
-    target = {'subject': '03', 'run': 4, 'suffix': 'sekret', 'desc': 'bleargh'}
-    assert target == parse_file_entities(filename, entities=entities)
-
-
 # XXX 0.14: Add dot to extension (difficult to parametrize with module-scoped fixture)
 def test_parse_file_entities_from_layout(layout_synthetic):
     layout = layout_synthetic
@@ -552,20 +526,6 @@ def test_deriv_indexing():
     assert not layout.get(scope='nonexistent')
 
 
-def test_add_config_paths():
-    bids_dir = dirname(bids.__file__)
-    bids_json = os.path.join(bids_dir, 'layout', 'config', 'bids.json')
-    with pytest.raises(ConfigError) as exc:
-        add_config_paths(test_config1='nonexistentpath.json')
-    assert str(exc.value).startswith('Configuration file')
-    with pytest.raises(ConfigError) as exc:
-        add_config_paths(bids=bids_json)
-    assert str(exc.value).startswith("Configuration 'bids' already")
-    add_config_paths(dummy=bids_json)
-    config = Config.load('dummy')
-    assert 'subject' in config.entities
-
-
 def test_layout_in_scope(layout_ds005, layout_ds005_derivs):
     assert layout_ds005._in_scope(['all'])
     assert layout_ds005._in_scope('raw')
@@ -629,7 +589,7 @@ def test_layout_save(tmp_path, layout_7t_trt):
     layout_7t_trt.save(str(tmp_path / "f.sqlite"),
                        replace_connection=False)
     data_dir = join(get_test_data_path(), '7t_trt')
-    layout = BIDSLayout(data_dir, database_path=str(tmp_path))
+    layout = BIDSLayout(data_dir, database_path=str(tmp_path / "f.sqlite"))
     oldfies = set(layout_7t_trt.get(suffix='events', return_type='file'))
     newfies = set(layout.get(suffix='events', return_type='file'))
     assert oldfies == newfies
@@ -639,7 +599,6 @@ def test_indexing_tag_conflict():
     data_dir = join(get_test_data_path(), 'ds005_conflict')
     with pytest.raises(BIDSValidationError) as exc:
         layout = BIDSLayout(data_dir)
-        print(exc.value.message)
         assert exc.value.message.startswith("Conflicting values found")
         assert 'run' in exc.value.message
 
@@ -714,4 +673,6 @@ def test_load_layout(layout_synthetic_nodb, db_dir):
     reloaded = BIDSLayout.load(db_path)
     assert sorted(layout_synthetic_nodb.get(return_type='file')) == \
         sorted(reloaded.get(return_type='file'))
-    assert layout_synthetic_nodb._init_args == reloaded._init_args
+    cm1 = layout_synthetic_nodb.connection_manager
+    cm2 = reloaded.connection_manager
+    assert cm1.init_args == cm2.init_args
diff --git a/bids/layout/tests/test_models.py b/bids/layout/tests/test_models.py
index 770999fd..a2a18e6f 100644
--- a/bids/layout/tests/test_models.py
+++ b/bids/layout/tests/test_models.py
@@ -1,7 +1,8 @@
+"""Tests of functionality in the models module."""
+
 import sys
 import os
 import pytest
-import bids
 import copy
 import json
 from pathlib import Path
@@ -12,7 +13,6 @@ import numpy as np
 
 from bids.layout.models import (BIDSFile, Entity, Tag, Base, Config,
                                 FileAssociation, BIDSImageFile)
-from bids.layout import BIDSLayout
 from bids.tests import get_test_data_path
 
 
diff --git a/bids/layout/tests/test_path_building.py b/bids/layout/tests/test_path_building.py
index 560a70d4..04666d75 100644
--- a/bids/layout/tests/test_path_building.py
+++ b/bids/layout/tests/test_path_building.py
@@ -1,7 +1,10 @@
+"""Tests of path-building functionality."""
+
+from os.path import join
+from pathlib import Path
+
 import pytest
 from bids.layout import BIDSLayout
-from os.path import join, abspath, sep
-from pathlib import Path
 from bids.tests import get_test_data_path
 
 
diff --git a/bids/layout/tests/test_rootpath.py b/bids/layout/tests/test_rootpath.py
index d88ceba1..f8a72bca 100644
--- a/bids/layout/tests/test_rootpath.py
+++ b/bids/layout/tests/test_rootpath.py
@@ -1,11 +1,14 @@
+"""
+Test handling of pathlib Path file paths in place of old-style string type.
+"""
+
 import sys
 import pytest
 from pathlib import Path
+
 from bids.layout import BIDSLayout
 from bids.tests import get_test_data_path
-"""
-test handling of pathlib Path file paths in place of old-style string type
-"""
+
 
 TESTPATH = Path(get_test_data_path()).joinpath("ds005")
 TESTSTR = str(TESTPATH)
diff --git a/bids/layout/tests/test_utils.py b/bids/layout/tests/test_utils.py
index 603ecced..857984fd 100644
--- a/bids/layout/tests/test_utils.py
+++ b/bids/layout/tests/test_utils.py
@@ -1,5 +1,13 @@
+"""Test miscellaneous utilities."""
+
+import os
+
 import pytest
-from ..utils import BIDSMetadata
+import bids
+from bids.exceptions import ConfigError
+
+from ..models import Entity, Config
+from ..utils import BIDSMetadata, parse_file_entities, add_config_paths
 
 
 def test_bidsmetadata_class():
@@ -9,3 +17,49 @@ def test_bidsmetadata_class():
     assert "Metadata term 'Missing' unavailable for file fakefile." in str(err)
     md["Missing"] = 1
     assert md["Missing"] == 1
+
+
+@pytest.mark.parametrize("extension_initial_dot", (True, False))
+def test_parse_file_entities(mock_config, extension_initial_dot):
+    filename = '/sub-03_ses-07_run-4_desc-bleargh_sekret.nii.gz'
+
+    dot = '.' if extension_initial_dot else ''
+
+    # Test with entities taken from bids config
+    target = {'subject': '03', 'session': '07', 'run': 4, 'suffix': 'sekret',
+              'extension': dot + 'nii.gz'}
+    assert target == parse_file_entities(filename, config='bids')
+    config = Config.load('bids')
+    assert target == parse_file_entities(filename, config=[config])
+
+    # Test with entities taken from bids and derivatives config
+    target = {'subject': '03', 'session': '07', 'run': 4, 'suffix': 'sekret',
+              'desc': 'bleargh', 'extension': dot + 'nii.gz'}
+    assert target == parse_file_entities(filename)
+    assert target == parse_file_entities(
+        filename, config=['bids', 'derivatives'])
+
+    # Test with list of Entities
+    entities = [
+        Entity('subject', "[/\\\\]sub-([a-zA-Z0-9]+)"),
+        Entity('run', "[_/\\\\]run-0*(\\d+)", dtype=int),
+        Entity('suffix', "[._]*([a-zA-Z0-9]*?)\\.[^/\\\\]+$"),
+        Entity('desc', "desc-([a-zA-Z0-9]+)"),
+    ]
+    # Leave out session to distinguish from previous test target
+    target = {'subject': '03', 'run': 4, 'suffix': 'sekret', 'desc': 'bleargh'}
+    assert target == parse_file_entities(filename, entities=entities)
+
+
+def test_add_config_paths():
+    bids_dir = os.path.dirname(bids.__file__)
+    bids_json = os.path.join(bids_dir, 'layout', 'config', 'bids.json')
+    with pytest.raises(ConfigError) as exc:
+        add_config_paths(test_config1='nonexistentpath.json')
+    assert str(exc.value).startswith('Configuration file')
+    with pytest.raises(ConfigError) as exc:
+        add_config_paths(bids=bids_json)
+    assert str(exc.value).startswith("Configuration 'bids' already")
+    add_config_paths(dummy=bids_json)
+    config = Config.load('dummy')
+    assert 'subject' in config.entities
diff --git a/bids/layout/tests/test_validation.py b/bids/layout/tests/test_validation.py
index 0c73faf2..0cb7a78d 100644
--- a/bids/layout/tests/test_validation.py
+++ b/bids/layout/tests/test_validation.py
@@ -1,10 +1,11 @@
-""" Tests of BIDS-specific functionality. Generic tests of core grabbit
-functionality should go in the grabbit package. """
+"""Tests of BIDSValidator functionality."""
+
+from os.path import join, dirname, abspath
 
 import pytest
+
 from bids_validator import BIDSValidator
 from bids.layout import BIDSLayout
-from os.path import join, dirname, abspath
 from bids.tests import get_test_data_path
 
 
diff --git a/bids/layout/tests/test_writing.py b/bids/layout/tests/test_writing.py
index ac14026c..ecf12e0d 100644
--- a/bids/layout/tests/test_writing.py
+++ b/bids/layout/tests/test_writing.py
@@ -1,16 +1,18 @@
-import pytest
-import os
+"""Tests related to file-writing functionality."""
+
 import shutil
+import os
 from os.path import join, exists, islink, dirname
 
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
 from bids.layout.writing import build_path, _PATTERN_FIND
 from bids.tests import get_test_data_path
 from bids import BIDSLayout
 from bids.layout.models import BIDSFile, Entity, Tag, Base
 
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
 
 @pytest.fixture
 def writable_file(tmpdir):