Repository: IAMconsortium/pyam (python)

Issue:
allow explicit `regexp` in filter arguments
Currently, using strings for filtering in the `filter()` function implements a pseudo-syntax, where `*` is used as wildcard and other regexp-arguments are escaped to have a "close-to-string" user experience. Hpwever, for expert users, we should add a feature to explicitly allow regexp (i.e., do not use the pseudo-syntax in teh `pattern_match()`).

Pull request description:
extend pseudo-regexp in `pattern_match()` for `$`
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

# Description of PR

This PR adds the `$` symbol to the list of characters escaped in the `pattern_match()` function, to allow filtering for e.g. `unit='US$'`.

It also adds the option to functions using `_apply_filters()` to pass `regexp=True` as a kwarg, overriding the pseudo-regexp syntax in `pattern_match()`. Closes #79

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 4127093..3dbce36 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 
 # Next Release
 
+- (#92)[https://github.com/IAMconsortium/pyam/pull/92] Adding `$` to the pseudo-regexp syntax in `pattern_match()`, adds override option
 - (#90)[https://github.com/IAMconsortium/pyam/pull/90] Adding a function to `set_panel_label()` as part of the plotting library
 - (#87)[https://github.com/IAMconsortium/pyam/pull/87] Extending `rename()` to work with model and scenario names
 - (#85)[https://github.com/IAMconsortium/pyam/pull/85] Improved functionality for importing metadata and bugfix for filtering for strings if `nan` values exist in metadata
diff --git a/pyam/core.py b/pyam/core.py
index ab11733..f338d58 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -553,6 +553,7 @@ class IamDataFrame(object):
              - 'year': takes an integer, a list of integers or a range
                 note that the last year of a range is not included,
                 so ``range(2010,2015)`` is interpreted as ``[2010, ..., 2014]``
+            - 'regexp=True' overrides pseudo-regexp syntax in `pattern_match()`
         """
         if filters is not None:
             warnings.warn(
@@ -829,28 +830,42 @@ def _aggregate_by_variables(df, variables, units=None):
 
 
 def _apply_filters(data, meta, filters):
+    """Applies filters to the data and meta tables of an IamDataFrame.
+
+    Parametersp
+    ----------
+    data: pd.DataFrame
+        data table of an IamDataFrame
+    meta: pd.DataFrame
+        meta table of an IamDataFrame
+    filters: dict
+        dictionary of filters ({col: values}}); uses a pseudo-regexp syntax by
+        default, but accepts `regexp: True` to use direct regexp
+    """
+    regexp = filters.pop('regexp', False)
     keep = np.array([True] * len(data))
 
     # filter by columns and list of values
     for col, values in filters.items():
         if col in meta.columns:
-            matches = pattern_match(meta[col], values)
+            matches = pattern_match(meta[col], values, regexp=regexp)
             cat_idx = meta[matches].index
             keep_col = data[META_IDX].set_index(META_IDX).index.isin(cat_idx)
 
         elif col in ['model', 'scenario', 'region', 'unit']:
-            keep_col = pattern_match(data[col], values)
+            keep_col = pattern_match(data[col], values, regexp=regexp)
 
         elif col == 'variable':
-            level = filters['level'] if 'level' in filters.keys() else None
-            keep_col = pattern_match(data[col], values, level)
+            level = filters['level'] if 'level' in filters else None
+            keep_col = pattern_match(data[col], values, level, regexp)
 
         elif col == 'year':
             keep_col = years_match(data[col], values)
 
         elif col == 'level':
             if 'variable' not in filters.keys():
-                keep_col = pattern_match(data['variable'], '*', level=values)
+                keep_col = pattern_match(data['variable'], '*', values,
+                                         regexp=regexp)
             else:
                 continue
         else:
diff --git a/pyam/utils.py b/pyam/utils.py
index a830439..4f6d2a7 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -218,10 +218,10 @@ def find_depth(data, s, level):
     return list(map(apply_test, data))
 
 
-def pattern_match(data, values, level=None):
+def pattern_match(data, values, level=None, regexp=False):
     """
-    matching of model/scenario names, variables, regions, and categories
-    to pseudo-regex for filtering by columns (str, int, bool)
+    matching of model/scenario names, variables, regions, and meta columns to
+    pseudo-regex (if `regexp == False`) for filtering (str, int, bool)
     """
     matches = np.array([False] * len(data))
     if not isinstance(values, collections.Iterable) or isstr(values):
@@ -233,15 +233,17 @@ def pattern_match(data, values, level=None):
 
     for s in values:
         if isstr(s):
-            regexp = (str(s)
-                      .replace('|', '\\|')
-                      .replace('.', '\.')  # `.` has to be replaced before `*`
-                      .replace('*', '.*')
-                      .replace('+', '\+')
-                      .replace('(', '\(')
-                      .replace(')', '\)')
-                      ) + "$"
-            pattern = re.compile(regexp)
+            _regexp = (str(s)
+                       .replace('|', '\\|')
+                       .replace('.', '\.')  # `.` has to be replaced before `*`
+                       .replace('*', '.*')
+                       .replace('+', '\+')
+                       .replace('(', '\(')
+                       .replace(')', '\)')
+                       .replace('$', '\\$')
+                       ) + "$"
+            pattern = re.compile(_regexp if not regexp else s)
+
             subset = filter(pattern.match, _data)
             depth = True if level is None else find_depth(_data, s, level)
             matches |= (_data.isin(subset) & depth)


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 43ad2ba..fd229c1 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -121,6 +121,11 @@ def test_filter_keep_false(meta_df):
     npt.assert_array_equal(obs, [1, 6, 3])
 
 
+def test_filter_by_regexp(meta_df):
+    obs = meta_df.filter(scenario='a_scenari.$', regexp=True)
+    assert obs['scenario'].unique() == 'a_scenario'
+
+
 def test_timeseries(test_df):
     dct = {'model': ['a_model'] * 2, 'scenario': ['a_scenario'] * 2,
            'years': [2005, 2010], 'value': [1, 6]}
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 3430e1a..2feb7c4 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -81,5 +81,24 @@ def test_pattern_match_brackets():
     obs = utils.pattern_match(data, values)
     exp = [True, False]
 
-    print(obs)
+    assert (obs == exp).all()
+
+
+def test_pattern_match_dollar():
+    data = pd.Series(['foo$bar', 'foo'])
+    values = ['foo$bar']
+
+    obs = utils.pattern_match(data, values)
+    exp = [True, False]
+
+    assert (obs == exp).all()
+
+
+def test_pattern_regexp():
+    data = pd.Series(['foo', 'foa', 'foo$'])
+    values = ['fo.$']
+
+    obs = utils.pattern_match(data, values, regexp=True)
+    exp = [True, True, False]
+
     assert (obs == exp).all()