Repository: IAMconsortium/pyam
Pull request #87 (2018-09-10 17:45:53)

Allow `rename()` for model and scenarios
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

# Description

This PR extends the `rename()` function to also work for the `model` and `scenario` names. In contrast to renaming of `variable` or `region`, renaming by `model` and `scenario` enforces a unique `meta.index` and will raise an error if renaming leads to duplicates of model-scenario combination.



gidden: lgtm, can you rebase/update on current `master` and rm the prints, then should be good to go!

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 52ba58d..64fd5c8 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 
 # Next Release
 
+- (#87)[https://github.com/IAMconsortium/pyam/pull/87] Extending `rename()` to work with model and scenario names
 - (#85)[https://github.com/IAMconsortium/pyam/pull/85] Improved functionality for importing metadata and bugfix for filtering for strings if `nan` values exist in metadata
 - (#83)[https://github.com/IAMconsortium/pyam/pull/83] Extending `filter_by_meta()` to work with non-matching indices between `df` and `data
 - (#81)[https://github.com/IAMconsortium/pyam/pull/81] Bug-fix when using `set_meta()` with unnamed pd.Series and no `name` kwarg
diff --git a/pyam/core.py b/pyam/core.py
index de1857c..ab11733 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -432,7 +432,9 @@ class IamDataFrame(object):
             return df
 
     def rename(self, mapping, inplace=False):
-        """Rename and aggregate column entries using groupby.sum()
+        """Rename and aggregate column entries using `groupby.sum()` on values.
+        When renaming models or scenarios, the uniqueness of the index must be
+        maintained, and the function will raise an error otherwise.
 
         Parameters
         ----------
@@ -445,10 +447,17 @@ class IamDataFrame(object):
             if True, do operation inplace and return None
         """
         ret = copy.deepcopy(self) if not inplace else self
-        for col in mapping:
-            if col not in ['region', 'variable', 'unit']:
-                raise ValueError('renaming by {} not supported!'.format(col))
-            ret.data.loc[:, col] = self.data.loc[:, col].replace(mapping[col])
+        for col, _mapping in mapping.items():
+            if col in ['model', 'scenario']:
+                index = pd.DataFrame(index=ret.meta.index).reset_index()
+                index.loc[:, col] = index.loc[:, col].replace(_mapping)
+                if index.duplicated().any():
+                    raise ValueError('Renaming to non-unique {} index!'
+                                     .format(col))
+                ret.meta.index = index.set_index(META_IDX).index
+            elif col not in ['region', 'variable', 'unit']:
+                raise ValueError('Renaming by {} not supported!'.format(col))
+            ret.data.loc[:, col] = ret.data.loc[:, col].replace(_mapping)
 
         ret.data = ret.data.groupby(LONG_IDX).sum().reset_index()
         if not inplace:


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index b03b1ef..43ad2ba 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -7,7 +7,7 @@ import pandas as pd
 from numpy import testing as npt
 
 from pyam import IamDataFrame, plotting, validate, categorize, \
-    require_variable, check_aggregate, filter_by_meta, META_IDX
+    require_variable, check_aggregate, filter_by_meta, META_IDX, IAMC_IDX
 from pyam.core import _meta_idx
 
 from conftest import TEST_DATA_DIR
@@ -549,7 +549,7 @@ def test_48b():
     pd.testing.assert_frame_equal(obs, exp, check_index_type=False)
 
 
-def test_rename():
+def test_rename_variable():
     df = IamDataFrame(pd.DataFrame([
         ['model', 'scen', 'SST', 'test_1', 'unit', 1, 5],
         ['model', 'scen', 'SDN', 'test_2', 'unit', 2, 6],
@@ -572,6 +572,35 @@ def test_rename():
     pd.testing.assert_frame_equal(obs, exp, check_index_type=False)
 
 
+def test_rename_index_fail(meta_df):
+    mapping = {'scenario': {'a_scenario': 'a_scenario2'}}
+    pytest.raises(ValueError, meta_df.rename, mapping)
+
+
+def test_rename_index(meta_df):
+    mapping = {'model': {'a_model': 'b_model'},
+               'scenario': {'a_scenario': 'b_scen'}}
+    obs = meta_df.rename(mapping)
+
+    # test data changes
+    exp = pd.DataFrame([
+        ['b_model', 'b_scen', 'World', 'Primary Energy', 'EJ/y', 1., 6.],
+        ['b_model', 'b_scen', 'World', 'Primary Energy|Coal', 'EJ/y', .5, 3.],
+        ['b_model', 'a_scenario2', 'World', 'Primary Energy', 'EJ/y', 2., 7.],
+    ], columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
+    ).set_index(IAMC_IDX).sort_index()
+    exp.columns = exp.columns.map(int)
+    pd.testing.assert_frame_equal(obs.timeseries().sort_index(), exp)
+
+    # test meta changes
+    exp = pd.DataFrame([
+        ['b_model', 'b_scen', False],
+        ['b_model', 'a_scenario2', False],
+    ], columns=['model', 'scenario', 'exclude']
+    ).set_index(META_IDX)
+    pd.testing.assert_frame_equal(obs.meta, exp)
+
+
 def test_convert_unit():
     df = IamDataFrame(pd.DataFrame([
         ['model', 'scen', 'SST', 'test_1', 'A', 1, 5],
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index 48a8b3f..127c206 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -24,7 +24,6 @@ from pyam import IamDataFrame, plotting, run_control, reset_rc_defaults
 from conftest import IMAGE_BASELINE_DIR, TEST_DATA_DIR
 
 IS_WINDOWS = os.name == 'nt'
-WINDOWS_REASON = "CI on Windows fails with silly dimension difference"
 TOLERANCE = 6 if IS_WINDOWS else 2
 
 MPL_KWARGS = {
@@ -42,7 +41,6 @@ def update_run_control(update):
     reset_rc_defaults()
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_plot(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -50,7 +48,6 @@ def test_line_plot(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_plot_dict_legend(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -59,7 +56,6 @@ def test_line_plot_dict_legend(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_no_legend(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -67,7 +63,6 @@ def test_line_no_legend(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_color(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -75,7 +70,6 @@ def test_line_color(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_marker_legend(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -83,7 +77,6 @@ def test_line_marker_legend(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_rm_legend_label(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -92,7 +85,6 @@ def test_line_rm_legend_label(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_linestyle_legend(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -100,7 +92,6 @@ def test_line_linestyle_legend(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_single_color(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -108,8 +99,6 @@ def test_line_single_color(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS,
-                    reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_filter_title(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -118,7 +107,6 @@ def test_line_filter_title(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_update_rc(plot_df):
     with update_run_control({'color': {'model': {'test_model1': 'cyan'}}}):
@@ -127,7 +115,6 @@ def test_line_update_rc(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_plot_1_var(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -138,7 +125,6 @@ def test_line_plot_1_var(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_line_plot_2_vars(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -149,7 +135,6 @@ def test_line_plot_2_vars(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region():
@@ -165,7 +150,6 @@ def test_region():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_cbar():
@@ -181,7 +165,6 @@ def test_region_cbar():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_cbar_args():
@@ -197,7 +180,6 @@ def test_region_cbar_args():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_vmin_vmax():
@@ -215,7 +197,6 @@ def test_region_vmin_vmax():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_cmap():
@@ -232,7 +213,6 @@ def test_region_cmap():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_crs():
@@ -249,7 +229,6 @@ def test_region_crs():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_map_regions():
@@ -266,7 +245,6 @@ def test_region_map_regions():
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.skipif(not has_cartopy, reason="requires cartopy")
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_region_map_regions_legend():
@@ -298,7 +276,6 @@ def test_bar_plot(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_bar_plot_h(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -374,7 +351,6 @@ def test_pie_plot_other(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_stack_plot(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))
@@ -385,7 +361,6 @@ def test_stack_plot(plot_df):
     return fig
 
 
-@pytest.mark.skipif(IS_WINDOWS, reason=WINDOWS_REASON)
 @pytest.mark.mpl_image_compare(**MPL_KWARGS)
 def test_stack_plot_other(plot_df):
     fig, ax = plt.subplots(figsize=(8, 8))