Repository: bids-standard/pybids

Issue:
combination of regex_search and run makes sql barf
This looks like a fun one.
The code worked in 0.9.3, but is broken in 0.9.4.

If you change the line in the binder notebook from:
```
# Retrieve filenames of all BOLD runs for subject 01
layout.get(subject='01', extension='nii.gz', suffix='bold', return_type='filename')
```
to:
```
# Retrieve filenames of all BOLD runs for subject 01
layout.get(subject='01', extension='nii.gz', suffix='bold', regex_search=True, run=1, return_type='filename')
```

you will get this error:
```
---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1248                     self.dialect.do_execute(
-> 1249                         cursor, statement, parameters, context
   1250                     )

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 

OperationalError: user-defined function raised exception

The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
<ipython-input-6-7db3897f9b6e> in <module>
      1 # Retrieve filenames of all BOLD runs for subject 01
----> 2 layout.get(subject='01', extension='nii.gz', suffix='bold', regex_search=True, run=1, return_type='filename')

/srv/conda/envs/notebook/lib/python3.7/site-packages/bids/layout/layout.py in get(self, return_type, target, scope, regex_search, absolute_paths, drop_invalid_filters, **filters)
    728             query = query.options(joinedload(BIDSFile.tags)
    729                                   .joinedload(Tag.entity))
--> 730             results.extend(query.all())
    731 
    732         # Convert to relative paths if needed

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in all(self)
   3176 
   3177         """
-> 3178         return list(self)
   3179 
   3180     @_generative(_no_clauseelement_condition)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in __iter__(self)
   3332         if self._autoflush and not self._populate_existing:
   3333             self.session._autoflush()
-> 3334         return self._execute_and_instances(context)
   3335 
   3336     def __str__(self):

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in _execute_and_instances(self, querycontext)
   3357         )
   3358 
-> 3359         result = conn.execute(querycontext.statement, self._params)
   3360         return loading.instances(querycontext.query, result, querycontext)
   3361 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
    986             raise exc.ObjectNotExecutableError(object_)
    987         else:
--> 988             return meth(self, multiparams, params)
    989 
    990     def _execute_function(self, func, multiparams, params):

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/sql/elements.py in _execute_on_connection(self, connection, multiparams, params)
    285     def _execute_on_connection(self, connection, multiparams, params):
    286         if self.supports_execution:
--> 287             return connection._execute_clauseelement(self, multiparams, params)
    288         else:
    289             raise exc.ObjectNotExecutableError(self)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_clauseelement(self, elem, multiparams, params)
   1105             distilled_params,
   1106             compiled_sql,
-> 1107             distilled_params,
   1108         )
   1109         if self._has_events or self.engine._has_events:

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1251         except BaseException as e:
   1252             self._handle_dbapi_exception(
-> 1253                 e, statement, parameters, cursor, context
   1254             )
   1255 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1471                 util.raise_from_cause(newraise, exc_info)
   1472             elif should_wrap:
-> 1473                 util.raise_from_cause(sqlalchemy_exception, exc_info)
   1474             else:
   1475                 util.reraise(*exc_info)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/util/compat.py in raise_from_cause(exception, exc_info)
    396     exc_type, exc_value, exc_tb = exc_info
    397     cause = exc_value if exc_value is not exception else None
--> 398     reraise(type(exception), exception, tb=exc_tb, cause=cause)
    399 
    400 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/util/compat.py in reraise(tp, value, tb, cause)
    150             value.__cause__ = cause
    151         if value.__traceback__ is not tb:
--> 152             raise value.with_traceback(tb)
    153         raise value
    154 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1247                 if not evt_handled:
   1248                     self.dialect.do_execute(
-> 1249                         cursor, statement, parameters, context
   1250                     )
   1251         except BaseException as e:

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    550 
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 
    554     def do_execute_no_params(self, cursor, statement, context=None):

OperationalError: (sqlite3.OperationalError) user-defined function raised exception
[SQL: SELECT files.path AS files_path, files.filename AS files_filename, files.dirname AS files_dirname, files.is_dir AS files_is_dir, files.class_ AS files_class_, entities_1.name AS entities_1_name, entities_1.mandatory AS entities_1_mandatory, entities_1.pattern AS entities_1_pattern, entities_1.directory AS entities_1_directory, entities_1.is_metadata AS entities_1_is_metadata, entities_1._dtype AS entities_1__dtype, tags_1.file_path AS tags_1_file_path, tags_1.entity_name AS tags_1_entity_name, tags_1._value AS tags_1__value, tags_1._dtype AS tags_1__dtype 
FROM files JOIN tags ON files.path = tags.file_path LEFT OUTER JOIN tags AS tags_1 ON files.path = tags_1.file_path LEFT OUTER JOIN entities AS entities_1 ON entities_1.name = tags_1.entity_name 
WHERE files.is_dir = 0 AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?)))]
[parameters: ('subject', '01', 'extension', 'nii.gz', 'suffix', 'bold', 'run', 1)]
(Background on this error at: http://sqlalche.me/e/e3q8)
```

This is a relevant query for NiBetaSeries since we [want to only select certain files in the derivatives directory](https://github.com/HBClab/NiBetaSeries/blob/604622a7a76c37c2aeecfb791cf34950b60792dd/src/nibetaseries/workflows/utils.py#L19-L39), and so we do this using the `desc` keyword being set to anything. I see I should change this to `scope: derivatives` for my use case, but there may be others out there that want to filter based on `run` and `regex_search` with another entity.

ref HBClab/NiBetaSeries#204
combination of regex_search and run makes sql barf
This looks like a fun one.
The code worked in 0.9.3, but is broken in 0.9.4.

If you change the line in the binder notebook from:
```
# Retrieve filenames of all BOLD runs for subject 01
layout.get(subject='01', extension='nii.gz', suffix='bold', return_type='filename')
```
to:
```
# Retrieve filenames of all BOLD runs for subject 01
layout.get(subject='01', extension='nii.gz', suffix='bold', regex_search=True, run=1, return_type='filename')
```

you will get this error:
```
---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1248                     self.dialect.do_execute(
-> 1249                         cursor, statement, parameters, context
   1250                     )

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 

OperationalError: user-defined function raised exception

The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
<ipython-input-6-7db3897f9b6e> in <module>
      1 # Retrieve filenames of all BOLD runs for subject 01
----> 2 layout.get(subject='01', extension='nii.gz', suffix='bold', regex_search=True, run=1, return_type='filename')

/srv/conda/envs/notebook/lib/python3.7/site-packages/bids/layout/layout.py in get(self, return_type, target, scope, regex_search, absolute_paths, drop_invalid_filters, **filters)
    728             query = query.options(joinedload(BIDSFile.tags)
    729                                   .joinedload(Tag.entity))
--> 730             results.extend(query.all())
    731 
    732         # Convert to relative paths if needed

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in all(self)
   3176 
   3177         """
-> 3178         return list(self)
   3179 
   3180     @_generative(_no_clauseelement_condition)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in __iter__(self)
   3332         if self._autoflush and not self._populate_existing:
   3333             self.session._autoflush()
-> 3334         return self._execute_and_instances(context)
   3335 
   3336     def __str__(self):

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/orm/query.py in _execute_and_instances(self, querycontext)
   3357         )
   3358 
-> 3359         result = conn.execute(querycontext.statement, self._params)
   3360         return loading.instances(querycontext.query, result, querycontext)
   3361 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in execute(self, object_, *multiparams, **params)
    986             raise exc.ObjectNotExecutableError(object_)
    987         else:
--> 988             return meth(self, multiparams, params)
    989 
    990     def _execute_function(self, func, multiparams, params):

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/sql/elements.py in _execute_on_connection(self, connection, multiparams, params)
    285     def _execute_on_connection(self, connection, multiparams, params):
    286         if self.supports_execution:
--> 287             return connection._execute_clauseelement(self, multiparams, params)
    288         else:
    289             raise exc.ObjectNotExecutableError(self)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_clauseelement(self, elem, multiparams, params)
   1105             distilled_params,
   1106             compiled_sql,
-> 1107             distilled_params,
   1108         )
   1109         if self._has_events or self.engine._has_events:

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1251         except BaseException as e:
   1252             self._handle_dbapi_exception(
-> 1253                 e, statement, parameters, cursor, context
   1254             )
   1255 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   1471                 util.raise_from_cause(newraise, exc_info)
   1472             elif should_wrap:
-> 1473                 util.raise_from_cause(sqlalchemy_exception, exc_info)
   1474             else:
   1475                 util.reraise(*exc_info)

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/util/compat.py in raise_from_cause(exception, exc_info)
    396     exc_type, exc_value, exc_tb = exc_info
    397     cause = exc_value if exc_value is not exception else None
--> 398     reraise(type(exception), exception, tb=exc_tb, cause=cause)
    399 
    400 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/util/compat.py in reraise(tp, value, tb, cause)
    150             value.__cause__ = cause
    151         if value.__traceback__ is not tb:
--> 152             raise value.with_traceback(tb)
    153         raise value
    154 

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/base.py in _execute_context(self, dialect, constructor, statement, parameters, *args)
   1247                 if not evt_handled:
   1248                     self.dialect.do_execute(
-> 1249                         cursor, statement, parameters, context
   1250                     )
   1251         except BaseException as e:

/srv/conda/envs/notebook/lib/python3.7/site-packages/sqlalchemy/engine/default.py in do_execute(self, cursor, statement, parameters, context)
    550 
    551     def do_execute(self, cursor, statement, parameters, context=None):
--> 552         cursor.execute(statement, parameters)
    553 
    554     def do_execute_no_params(self, cursor, statement, context=None):

OperationalError: (sqlite3.OperationalError) user-defined function raised exception
[SQL: SELECT files.path AS files_path, files.filename AS files_filename, files.dirname AS files_dirname, files.is_dir AS files_is_dir, files.class_ AS files_class_, entities_1.name AS entities_1_name, entities_1.mandatory AS entities_1_mandatory, entities_1.pattern AS entities_1_pattern, entities_1.directory AS entities_1_directory, entities_1.is_metadata AS entities_1_is_metadata, entities_1._dtype AS entities_1__dtype, tags_1.file_path AS tags_1_file_path, tags_1.entity_name AS tags_1_entity_name, tags_1._value AS tags_1__value, tags_1._dtype AS tags_1__dtype 
FROM files JOIN tags ON files.path = tags.file_path LEFT OUTER JOIN tags AS tags_1 ON files.path = tags_1.file_path LEFT OUTER JOIN entities AS entities_1 ON entities_1.name = tags_1.entity_name 
WHERE files.is_dir = 0 AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?))) AND (EXISTS (SELECT 1 
FROM tags 
WHERE files.path = tags.file_path AND tags.entity_name = ? AND (tags._value REGEXP ?)))]
[parameters: ('subject', '01', 'extension', 'nii.gz', 'suffix', 'bold', 'run', 1)]
(Background on this error at: http://sqlalche.me/e/e3q8)
```

This is a relevant query for NiBetaSeries since we [want to only select certain files in the derivatives directory](https://github.com/HBClab/NiBetaSeries/blob/604622a7a76c37c2aeecfb791cf34950b60792dd/src/nibetaseries/workflows/utils.py#L19-L39), and so we do this using the `desc` keyword being set to anything. I see I should change this to `scope: derivatives` for my use case, but there may be others out there that want to filter based on `run` and `regex_search` with another entity.

ref HBClab/NiBetaSeries#204


Discussion:
As a workaround (and we should probably mention this in the docstring), you can set `desc=None`, which is a way of specifying that the key must be present, but can be set to anything. Assuming that solves your immediate problem, we should still leave this open until I (or someone else) can take a closer look. Thanks for reporting!
I suspect what's happening here is that as a result of automatically enforcing the `dtype` for each `Entity`, the run index is being converted to an `int`, which the custom `REGEXP` function doesn't like. If that's right, should be a simple fix (we can just force string dtype inside the custom function).
> As a workaround (and we should probably mention this in the docstring), you can set `desc=None`, which is a way of specifying that the key must be present

Oh man, it used to mean the key must not be present.
> Oh man, it used to mean the key must not be present.

Argh, you're right, it still means that. I'd remembered the opposite and just glanced at the code. I think what screwed me up is we used to have a `defined_entities` keyword or something like that. The latter only used to work for metadata, but now that we use a DB, it would be really easy to bring it back and apply to all entities. I'll open an issue separately from this.
As a workaround (and we should probably mention this in the docstring), you can set `desc=None`, which is a way of specifying that the key must be present, but can be set to anything. Assuming that solves your immediate problem, we should still leave this open until I (or someone else) can take a closer look. Thanks for reporting!
I suspect what's happening here is that as a result of automatically enforcing the `dtype` for each `Entity`, the run index is being converted to an `int`, which the custom `REGEXP` function doesn't like. If that's right, should be a simple fix (we can just force string dtype inside the custom function).
> As a workaround (and we should probably mention this in the docstring), you can set `desc=None`, which is a way of specifying that the key must be present

Oh man, it used to mean the key must not be present.
> Oh man, it used to mean the key must not be present.

Argh, you're right, it still means that. I'd remembered the opposite and just glanced at the code. I think what screwed me up is we used to have a `defined_entities` keyword or something like that. The latter only used to work for metadata, but now that we use a DB, it would be really easy to bring it back and apply to all entities. I'll open an issue separately from this.

Resolving patch:
diff --git a/bids/layout/layout.py b/bids/layout/layout.py
--- a/bids/layout/layout.py
+++ b/bids/layout/layout.py
@@ -829,7 +829,7 @@ def _build_file_query(self, **kwargs):
                         val_clause = sa.or_(*[Tag._value.op('REGEXP')(str(v))
                                               for v in val])
                     else:
-                        val_clause = Tag._value.op('REGEXP')(val)
+                        val_clause = Tag._value.op('REGEXP')(str(val))
                 else:
                     if isinstance(val, (list, tuple)):
                         val_clause = Tag._value.in_(val)


Test patch:
diff --git a/bids/layout/tests/test_layout.py b/bids/layout/tests/test_layout.py
--- a/bids/layout/tests/test_layout.py
+++ b/bids/layout/tests/test_layout.py
@@ -587,3 +587,33 @@ def test_get_with_wrong_dtypes(layout_7t_trt):
             l.get(run=[1, '15']))
     assert not l.get(run='not_numeric')
     assert l.get(session=1) == l.get(session='1')
+
+
+def test_get_with_regex_search(layout_7t_trt):
+    """ Tests that regex-based searching works. """
+    l = layout_7t_trt
+
+    # subject matches both '10' and '01'
+    results = l.get(subject='1', session='1', task='rest', suffix='bold',
+                    acquisition='fron.al', extension='nii.gz',
+                    regex_search=True)
+    assert len(results) == 2
+
+    # subject matches '10'
+    results = l.get(subject='^1', session='1', task='rest', suffix='bold',
+                    acquisition='fron.al', extension='nii.gz',
+                    regex_search=True, return_type='filename')
+    assert len(results) == 1
+    assert results[0].endswith('sub-10_ses-1_task-rest_acq-prefrontal_bold.nii.gz')
+
+
+def test_get_with_regex_search_bad_dtype(layout_7t_trt):
+    """ Tests that passing in a non-string dtype for an entity doesn't crash
+    regexp-based searching (i.e., that implicit conversion is done
+    appropriately). """
+    l = layout_7t_trt
+    results = l.get(subject='1', run=1, task='rest', suffix='bold',
+                    acquisition='fullbrain', extension='nii.gz',
+                    regex_search=True)
+    # Two runs (1 per session) for each of subjects '10' and '01'
+    assert len(results) == 4