Repository: IAMconsortium/pyam
Pull request #83 (2018-08-25 16:18:45)

Fix `filter_by_meta()` with nonmatching index
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

# Description

This PR extends the function `filter_by_meta()` to work as expected when the `IamDataFrame.index` does not match the `df.index`. Previously, entries were added from the `IamDataFrame.index` to the returned dataframe if the `IamDataFrame.index` contained additional `(model, scenario)` entries.

Now, the returned dataframe has the same size as the `data` arg (if no filter is applied, i.e., `col=None`, with `np.nan` for any rows where `(model, scenario) not in IamDataFrame.index`) or is smaller, if any filter is applied.



Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index e5b566a..b2d5d8d 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 
 # Next Release
 
+- (#83)[https://github.com/IAMconsortium/pyam/pull/83] Extending `filter_by_meta()` to work with non-matching indices between `df` and `data
 - (#81)[https://github.com/IAMconsortium/pyam/pull/81] Bug-fix when using `set_meta()` with unnamed pd.Series and no `name` kwarg
 - (#80)[https://github.com/IAMconsortium/pyam/pull/80] Extend the pseudo-regexp syntax for filtering in `pattern_match()`
 - (#73)[https://github.com/IAMconsortium/pyam/pull/73] Adds ability to remove labels for markers, colors, or linestyles
diff --git a/pyam/core.py b/pyam/core.py
index 680aa12..e801727 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -975,27 +975,35 @@ def filter_by_meta(data, df, join_meta=False, **kwargs):
     join_meta: bool, default False
         join selected columns from `df.meta` on `data`
     kwargs:
-        meta columns to be joined, where `col=...` applies filters
+        meta columns to be filtered/joined, where `col=...` applies filters
         by the given arguments (using `utils.pattern_match()`) and `col=None`
-        joins the column without filtering
+        joins the column without filtering (setting col to `np.nan`
+        if `(model, scenario) not in df.meta.index`)
     """
     if not set(META_IDX).issubset(data.index.names + list(data.columns)):
         raise ValueError('missing required index dimensions or columns!')
 
-    meta = df.meta[list(kwargs)].copy()
+    meta = pd.DataFrame(df.meta[list(kwargs)].copy())
 
     # filter meta by columns
     keep = np.array([True] * len(meta))
+    apply_filter = False
     for col, values in kwargs.items():
         if values is not None:
             keep_col = pattern_match(meta[col], values)
             keep &= keep_col
+            apply_filter = True
     meta = meta[keep]
 
     # set the data index to META_IDX and apply filtered meta index
     data = data.copy()
     idx = list(data.index.names) if not data.index.names == [None] else None
-    data = data.reset_index().set_index(META_IDX).loc[meta.index]
+    data = data.reset_index().set_index(META_IDX)
+    meta = meta.loc[meta.index.intersection(data.index)]
+    meta.index.names = META_IDX
+    if apply_filter:
+        data = data.loc[meta.index]
+    data.index.names = META_IDX
 
     # join meta (optional), reset index to format as input arg
     data = data.join(meta) if join_meta else data


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 35eee8e..af211eb 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -13,6 +13,21 @@ from pyam.core import _meta_idx
 from conftest import TEST_DATA_DIR
 
 
+df_filter_by_meta_matching_idx = pd.DataFrame([
+    ['a_model', 'a_scenario', 'a_region1', 1],
+    ['a_model', 'a_scenario', 'a_region2', 2],
+    ['a_model', 'a_scenario2', 'a_region3', 3],
+], columns=['model', 'scenario', 'region', 'col'])
+
+
+df_filter_by_meta_nonmatching_idx = pd.DataFrame([
+    ['a_model', 'a_scenario3', 'a_region1', 1, 2],
+    ['a_model', 'a_scenario3', 'a_region2', 2, 3],
+    ['a_model', 'a_scenario2', 'a_region3', 3, 4],
+], columns=['model', 'scenario', 'region', 2010, 2020]
+).set_index(['model', 'region'])
+
+
 def test_get_item(test_df):
     assert test_df['model'].unique() == ['a_model']
 
@@ -589,12 +604,7 @@ def test_convert_unit():
 
 
 def test_pd_filter_by_meta(meta_df):
-    data = pd.DataFrame([
-        ['a_model', 'a_scenario', 'a_region1', 1],
-        ['a_model', 'a_scenario', 'a_region2', 2],
-        ['a_model', 'a_scenario2', 'a_region3', 3],
-    ], columns=['model', 'scenario', 'region', 'col']
-    ).set_index(['model', 'region'])
+    data = df_filter_by_meta_matching_idx.set_index(['model', 'region'])
 
     meta_df.set_meta([True, False], 'boolean')
     meta_df.set_meta(0, 'integer')
@@ -611,11 +621,7 @@ def test_pd_filter_by_meta(meta_df):
 
 
 def test_pd_filter_by_meta_no_index(meta_df):
-    data = pd.DataFrame([
-        ['a_model', 'a_scenario', 'a_region1', 1],
-        ['a_model', 'a_scenario', 'a_region2', 2],
-        ['a_model', 'a_scenario2', 'a_region3', 3],
-    ], columns=['model', 'scenario', 'region', 'col'])
+    data = df_filter_by_meta_matching_idx
 
     meta_df.set_meta([True, False], 'boolean')
     meta_df.set_meta(0, 'int')
@@ -629,3 +635,29 @@ def test_pd_filter_by_meta_no_index(meta_df):
     exp['int'] = 0
 
     pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_pd_filter_by_meta_nonmatching_index(meta_df):
+    data = df_filter_by_meta_nonmatching_idx
+    meta_df.set_meta(['a', 'b'], 'string')
+
+    obs = filter_by_meta(data, meta_df, join_meta=True, string='b')
+    obs = obs.reindex(columns=['scenario', 2010, 2020, 'string'])
+
+    exp = data.iloc[2:3].copy()
+    exp['string'] = 'b'
+
+    pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_pd_join_by_meta_nonmatching_index(meta_df):
+    data = df_filter_by_meta_nonmatching_idx
+    meta_df.set_meta(['a', 'b'], 'string')
+
+    obs = filter_by_meta(data, meta_df, join_meta=True, string=None)
+    obs = obs.reindex(columns=['scenario', 2010, 2020, 'string'])
+
+    exp = data.copy()
+    exp['string'] = [np.nan, np.nan, 'b']
+
+    pd.testing.assert_frame_equal(obs.sort_index(level=1), exp)