Repository: IAMconsortium/pyam
Pull request #162 (2018-12-19 12:25:45)

Add a function to aggregate timeseries components
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

# Description of PR

This PR splits a "make aggregate" function out of the `check_aggregate()` function.

The PR also adapts the functions `check_aggregate()` and `check_aggregate_regions()` to be unit-specific.

For clean-up, all unit tests related to the `aggregate` features are migrated to an own file to reduce the clutter in `test_core.py`.

danielhuppmann: rebased
danielhuppmann: @znicholls, you implemented the function `check_aggregate_regions()` - do you have any objections to making these checks unit-specific?
coveralls: 
[![Coverage Status](https://coveralls.io/builds/20716744/badge)](https://coveralls.io/builds/20716744)

Coverage increased (+0.2%) to 83.257% when pulling **975ca9fa71071008153bd4c29cc4af269cfb1809 on danielhuppmann:compute_aggregate** into **dbefc911f6418d433d623c6e8b7be1cdf3f3aa1a on IAMconsortium:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/20716744/badge)](https://coveralls.io/builds/20716744)

Coverage increased (+0.2%) to 83.257% when pulling **975ca9fa71071008153bd4c29cc4af269cfb1809 on danielhuppmann:compute_aggregate** into **dbefc911f6418d433d623c6e8b7be1cdf3f3aa1a on IAMconsortium:master**.

danielhuppmann: @znicholls, seems that @gidden didn't have time any more before leaving for an extended Christmas trip to do a final review and merge - want to go ahead and squash it?

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 8f834b7..9a12caa 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 
 # Next Release
 
+- [#162](https://github.com/IAMconsortium/pyam/pull/162) Add a function to sum and append timeseries components to an aggregate variable
 - [#152](https://github.com/IAMconsortium/pyam/pull/152) Fix bug where scatter plots did not work with property metadata when using two variables (#136, #152)
 - [#151](https://github.com/IAMconsortium/pyam/pull/151) Fix bug where excel files were not being written on Windows and MacOSX (#149)
 - [#145](https://github.com/IAMconsortium/pyam/pull/145) Support full semantic and VCS-style versioning with `versioneer`
diff --git a/pyam/core.py b/pyam/core.py
index cfaf4ee..3940a08 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -35,6 +35,7 @@ from pyam.utils import (
     IAMC_IDX,
     SORT_IDX,
     LONG_IDX,
+    GROUP_IDX
 )
 from pyam.timeseries import fill_series
 
@@ -62,7 +63,7 @@ class IamDataFrame(object):
             https://github.com/IAMconsortium/pyam/issues
         """
         # import data from pd.DataFrame or read from source
-        if isinstance(data, pd.DataFrame):
+        if isinstance(data, pd.DataFrame) or isinstance(data, pd.Series):
             self.data = format_data(data.copy())
         elif has_ix and isinstance(data, ixmp.TimeSeries):
             self.data = read_ix(data, **kwargs)
@@ -533,6 +534,41 @@ class IamDataFrame(object):
         if not inplace:
             return ret
 
+    def aggregate(self, variable, components=None, units=None, append=False):
+        """Compute the aggregate of timeseries components or sub-categories
+
+        Parameters
+        ----------
+        variable: str
+            variable for which the aggregate should be computed
+        components: list of str, default None
+            list of variables, defaults to all sub-categories of `variable`
+        units: str or list of str, default None
+            filter variable and components for given unit(s)
+        append: bool
+            append the aggregate timeseries to `data` and return None,
+            else return aggregate timeseries
+        """
+        # default components to all variables one level below `variable`
+        if components is None:
+            var_list = pd.Series(self.data.variable.unique())
+            components = var_list[pattern_match(var_list,
+                                                '{}|*'.format(variable), 0)]
+
+        if not len(components):
+            msg = 'cannot aggregate {} because it has no components'
+            logger().info(msg.format(variable))
+
+            return
+
+        df_components = _aggregate_by_variables(self.data, components, units)
+
+        if append is True:
+            self.append(pd.concat([df_components], names=['variable'],
+                                  keys=[variable]), inplace=True)
+        else:
+            return df_components
+
     def check_aggregate(self, variable, components=None, units=None,
                         exclude_on_fail=False, multiplier=1, **kwargs):
         """Check whether the timeseries data match the aggregation
@@ -552,22 +588,15 @@ class IamDataFrame(object):
             factor when comparing variable and sum of components
         kwargs: passed to `np.isclose()`
         """
-        # default components to all variables one level below `variable`
-        if components is None:
-            var_list = pd.Series(self.data.variable.unique())
-            components = var_list[pattern_match(var_list,
-                                                '{}|*'.format(variable), 0)]
-
-        if not len(components):
-            msg = 'cannot check aggregate for {} because it has no components'
-            logger().info(msg.format(variable))
-
+        # compute aggregate from components, return None if no components
+        df_components = self.aggregate(variable, components, units)
+        if df_components is None:
             return
 
         # filter and groupby data, use `pd.Series.align` for matching index
         df_variable, df_components = (
             _aggregate_by_variables(self.data, variable, units)
-            .align(_aggregate_by_variables(self.data, components, units))
+            .align(df_components)
         )
 
         # use `np.isclose` for checking match
@@ -579,7 +608,7 @@ class IamDataFrame(object):
             logger().info(msg.format(variable, len(diff), len(df_variable)))
 
             if exclude_on_fail:
-                self._exclude_on_fail(diff.index.droplevel([2, 3]))
+                self._exclude_on_fail(diff.index.droplevel([2, 3, 4]))
 
             diff = pd.concat([diff], keys=[variable], names=['variable'])
 
@@ -645,8 +674,8 @@ class IamDataFrame(object):
             if not var_has_regional_info:
                 df_var_to_add = self.filter(
                     region=region, variable=var_to_add
-                ).data.groupby(REGION_IDX).sum()['value']
-                df_var_to_add.index = df_var_to_add.index.droplevel("variable")
+                ).data.groupby(REGION_IDX + ['unit']).sum()['value']
+                df_var_to_add.index = df_var_to_add.index.droplevel('variable')
 
                 if len(df_var_to_add):
                     df_components = df_components.add(df_var_to_add,
@@ -669,7 +698,7 @@ class IamDataFrame(object):
                 self._exclude_on_fail(diff.index.droplevel([2, 3]))
 
             diff = pd.concat([diff], keys=[region], names=['region'])
-
+            diff.index = diff.index.swaplevel(i=-1, j=-2)
             return diff.unstack().rename_axis(None, axis=1)
 
     def check_internal_consistency(self, **kwargs):
@@ -1075,7 +1104,7 @@ def _aggregate_by_variables(df, variables, units=None):
         units = [units] if isstr(units) else units
         df = df[df.unit.isin(units)]
 
-    return df.groupby(YEAR_IDX).sum()['value']
+    return df.groupby(GROUP_IDX).sum()['value']
 
 
 def _aggregate_by_regions(df, regions, units=None):
@@ -1086,7 +1115,7 @@ def _aggregate_by_regions(df, regions, units=None):
         units = [units] if isstr(units) else units
         df = df[df.unit.isin(units)]
 
-    return df.groupby(REGION_IDX).sum()['value']
+    return df.groupby(REGION_IDX + ['unit']).sum()['value']
 
 
 def _apply_filters(data, meta, filters):
diff --git a/pyam/utils.py b/pyam/utils.py
index 3bd0a61..ee87e63 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -28,6 +28,7 @@ REGION_IDX = ['model', 'scenario', 'variable', 'year']
 IAMC_IDX = ['model', 'scenario', 'region', 'variable', 'unit']
 SORT_IDX = ['model', 'scenario', 'variable', 'year', 'region']
 LONG_IDX = IAMC_IDX + ['year']
+GROUP_IDX = ['model', 'scenario', 'region', 'unit', 'year']
 
 # dictionary to translate column count to Excel column names
 NUMERIC_TO_STR = dict(zip(range(0, 702),
@@ -153,6 +154,9 @@ def read_files(fnames, *args, **kwargs):
 
 def format_data(df):
     """Convert an imported dataframe and check all required columns"""
+    if isinstance(df, pd.Series):
+        df = df.to_frame()
+
     # all lower case
     df.rename(columns={c: str(c).lower() for c in df.columns}, inplace=True)
 


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index e68f54b..7d6f5dd 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -271,193 +271,6 @@ def test_validate_top_level(meta_df):
     assert list(meta_df['exclude']) == [False, True]
 
 
-def test_check_aggregate_pass(check_aggregate_df):
-    obs = check_aggregate_df.filter(
-        scenario='a_scen'
-    ).check_aggregate('Primary Energy')
-    assert obs is None
-
-
-def test_check_aggregate_fail(meta_df):
-    obs = meta_df.check_aggregate('Primary Energy', exclude_on_fail=True)
-    assert len(obs.columns) == 2
-    assert obs.index.get_values()[0] == (
-        'Primary Energy', 'a_model', 'a_scenario', 'World'
-    )
-
-
-def test_check_aggregate_top_level(meta_df):
-    obs = check_aggregate(meta_df, variable='Primary Energy', year=2005)
-    assert len(obs.columns) == 1
-    assert obs.index.get_values()[0] == (
-        'Primary Energy', 'a_model', 'a_scenario', 'World'
-    )
-
-
-def test_df_check_aggregate_pass(check_aggregate_df):
-    obs = check_aggregate_df.check_aggregate('Primary Energy')
-    assert obs is None
-
-    for variable in check_aggregate_df.variables():
-        obs = check_aggregate_df.check_aggregate(variable)
-        assert obs is None
-
-
-def test_df_check_aggregate_regions_pass(check_aggregate_df):
-    obs = check_aggregate_df.check_aggregate_regions('Primary Energy')
-    assert obs is None
-
-    for variable in check_aggregate_df.variables():
-        obs = check_aggregate_df.check_aggregate_regions(variable)
-        assert obs is None
-
-
-def run_check_agg_fail(pyam_df, tweak_dict, test_type):
-    mr = pyam_df.data.model == tweak_dict['model']
-    sr = pyam_df.data.scenario == tweak_dict['scenario']
-    rr = pyam_df.data.region == tweak_dict['region']
-    vr = pyam_df.data.variable == tweak_dict['variable']
-    ur = pyam_df.data.unit == tweak_dict['unit']
-
-    row_to_tweak = mr & sr & rr & vr & ur
-    assert row_to_tweak.any()
-
-    pyam_df.data.value.iloc[np.where(row_to_tweak)[0]] *= 0.99
-
-    # the error variable is always the top level one
-    expected_index = tweak_dict
-    agg_test = test_type == 'aggregate'
-    region_world_only_contrib = test_type == 'region-world-only-contrib'
-    if agg_test or region_world_only_contrib:
-        expected_index['variable'] = '|'.join(
-            expected_index['variable'].split('|')[:2]
-        )
-    elif 'region' in test_type:
-        expected_index['region'] = 'World'
-
-    # units get dropped during aggregation and the index is a list
-    expected_index = [v for k, v in expected_index.items() if k != 'unit']
-
-    for variable in pyam_df.variables():
-        if test_type == 'aggregate':
-            obs = pyam_df.check_aggregate(
-                variable,
-            )
-        elif 'region' in test_type:
-            obs = pyam_df.check_aggregate_regions(
-                variable,
-            )
-
-        if obs is not None:
-            assert len(obs.columns) == 2
-            assert set(obs.index.get_values()[0]) == set(expected_index)
-
-
-def test_df_check_aggregate_fail(check_aggregate_df):
-    to_tweak = {
-        'model': 'IMG',
-        'scenario': 'a_scen_2',
-        'region': 'R5REF',
-        'variable': 'Emissions|CO2',
-        'unit': 'Mt CO2/yr',
-    }
-    run_check_agg_fail(check_aggregate_df, to_tweak, 'aggregate')
-
-
-def test_df_check_aggregate_fail_no_regions(check_aggregate_df):
-    to_tweak = {
-        'model': 'MSG-GLB',
-        'scenario': 'a_scen_2',
-        'region': 'World',
-        'variable': 'Emissions|C2F6|Solvents',
-        'unit': 'kt C2F6/yr',
-    }
-    run_check_agg_fail(check_aggregate_df, to_tweak, 'aggregate')
-
-
-def test_df_check_aggregate_region_fail(check_aggregate_df):
-    to_tweak = {
-        'model': 'IMG',
-        'scenario': 'a_scen_2',
-        'region': 'World',
-        'variable': 'Emissions|CO2',
-        'unit': 'Mt CO2/yr',
-    }
-
-    run_check_agg_fail(check_aggregate_df, to_tweak, 'region')
-
-
-def test_df_check_aggregate_region_fail_no_subsector(check_aggregate_df):
-    to_tweak = {
-        'model': 'MSG-GLB',
-        'scenario': 'a_scen_2',
-        'region': 'R5REF',
-        'variable': 'Emissions|CH4',
-        'unit': 'Mt CH4/yr',
-    }
-
-    run_check_agg_fail(check_aggregate_df, to_tweak, 'region')
-
-
-def test_df_check_aggregate_region_fail_world_only_var(check_aggregate_df):
-    to_tweak = {
-        'model': 'MSG-GLB',
-        'scenario': 'a_scen_2',
-        'region': 'World',
-        'variable': 'Emissions|CO2|Agg Agg',
-        'unit': 'Mt CO2/yr',
-    }
-
-    run_check_agg_fail(
-        check_aggregate_df, to_tweak, 'region-world-only-contrib'
-    )
-
-
-def test_df_check_aggregate_regions_errors(check_aggregate_regional_df):
-    # these tests should fail because our dataframe has continents and regions
-    # so checking without providing components leads to double counting and
-    # hence failure
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O', 'World'
-    )
-
-    assert len(obs.columns) == 2
-    assert obs.index.get_values()[0] == (
-        'World', 'AIM', 'cscen', 'Emissions|N2O'
-    )
-
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O', 'REUROPE'
-    )
-
-    assert len(obs.columns) == 2
-    assert obs.index.get_values()[0] == (
-        'REUROPE', 'AIM', 'cscen', 'Emissions|N2O'
-    )
-
-
-def test_df_check_aggregate_regions_components(check_aggregate_regional_df):
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O', 'World', components=['REUROPE', 'RASIA']
-    )
-    assert obs is None
-
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O|Solvents', 'World', components=['REUROPE', 'RASIA']
-    )
-    assert obs is None
-
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O', 'REUROPE', components=['Germany', 'UK']
-    )
-    assert obs is None
-
-    obs = check_aggregate_regional_df.check_aggregate_regions(
-        'Emissions|N2O|Transport', 'REUROPE', components=['Germany', 'UK']
-    )
-    assert obs is None
-
-
 def test_category_none(meta_df):
     meta_df.categorize('category', 'Testing', {'Primary Energy': {'up': 0.8}})
     assert 'category' not in meta_df.meta.columns
diff --git a/tests/test_feature_aggregate.py b/tests/test_feature_aggregate.py
new file mode 100644
index 0000000..fb017b0
--- /dev/null
+++ b/tests/test_feature_aggregate.py
@@ -0,0 +1,204 @@
+import numpy as np
+import pandas as pd
+from pyam import check_aggregate, IAMC_IDX
+
+
+def test_do_aggregate_append(meta_df):
+    meta_df.rename({'variable': {'Primary Energy': 'Primary Energy|Gas'}},
+                   inplace=True)
+    meta_df.aggregate('Primary Energy', append=True)
+    obs = meta_df.filter(variable='Primary Energy').timeseries()
+
+    exp = pd.DataFrame([
+        ['a_model', 'a_scenario', 'World', 'Primary Energy', 'EJ/y', 1.5, 9.],
+        ['a_model', 'a_scenario2', 'World', 'Primary Energy', 'EJ/y', 2, 7],
+    ],
+        columns=['model', 'scenario', 'region', 'variable', 'unit', 2005, 2010]
+    ).set_index(IAMC_IDX)
+    exp.columns = list(map(int, exp.columns))
+    pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_check_aggregate_pass(check_aggregate_df):
+    obs = check_aggregate_df.filter(
+        scenario='a_scen'
+    ).check_aggregate('Primary Energy')
+    assert obs is None
+
+
+def test_check_aggregate_fail(meta_df):
+    obs = meta_df.check_aggregate('Primary Energy', exclude_on_fail=True)
+    assert len(obs.columns) == 2
+    assert obs.index.get_values()[0] == (
+        'Primary Energy', 'a_model', 'a_scenario', 'World', 'EJ/y'
+    )
+
+
+def test_check_aggregate_top_level(meta_df):
+    obs = check_aggregate(meta_df, variable='Primary Energy', year=2005)
+    assert len(obs.columns) == 1
+    assert obs.index.get_values()[0] == (
+        'Primary Energy', 'a_model', 'a_scenario', 'World', 'EJ/y'
+    )
+
+
+def test_df_check_aggregate_pass(check_aggregate_df):
+    obs = check_aggregate_df.check_aggregate('Primary Energy')
+    assert obs is None
+
+    for variable in check_aggregate_df.variables():
+        obs = check_aggregate_df.check_aggregate(variable)
+        assert obs is None
+
+
+def test_df_check_aggregate_regions_pass(check_aggregate_df):
+    obs = check_aggregate_df.check_aggregate_regions('Primary Energy')
+    assert obs is None
+
+    for variable in check_aggregate_df.variables():
+        obs = check_aggregate_df.check_aggregate_regions(variable)
+        assert obs is None
+
+
+def run_check_agg_fail(pyam_df, tweak_dict, test_type):
+    mr = pyam_df.data.model == tweak_dict['model']
+    sr = pyam_df.data.scenario == tweak_dict['scenario']
+    rr = pyam_df.data.region == tweak_dict['region']
+    vr = pyam_df.data.variable == tweak_dict['variable']
+    ur = pyam_df.data.unit == tweak_dict['unit']
+
+    row_to_tweak = mr & sr & rr & vr & ur
+    assert row_to_tweak.any()
+
+    pyam_df.data.value.iloc[np.where(row_to_tweak)[0]] *= 0.99
+
+    # the error variable is always the top level one
+    expected_index = tweak_dict
+    agg_test = test_type == 'aggregate'
+    region_world_only_contrib = test_type == 'region-world-only-contrib'
+    if agg_test or region_world_only_contrib:
+        expected_index['variable'] = '|'.join(
+            expected_index['variable'].split('|')[:2]
+        )
+    elif 'region' in test_type:
+        expected_index['region'] = 'World'
+
+    # units get dropped during aggregation and the index is a list
+    expected_index = [v for k, v in expected_index.items()]
+
+    for variable in pyam_df.variables():
+        if test_type == 'aggregate':
+            obs = pyam_df.check_aggregate(
+                variable,
+            )
+        elif 'region' in test_type:
+            obs = pyam_df.check_aggregate_regions(
+                variable,
+            )
+
+        if obs is not None:
+            assert len(obs.columns) == 2
+            assert set(obs.index.get_values()[0]) == set(expected_index)
+
+
+def test_df_check_aggregate_fail(check_aggregate_df):
+    to_tweak = {
+        'model': 'IMG',
+        'scenario': 'a_scen_2',
+        'region': 'R5REF',
+        'variable': 'Emissions|CO2',
+        'unit': 'Mt CO2/yr',
+    }
+    run_check_agg_fail(check_aggregate_df, to_tweak, 'aggregate')
+
+
+def test_df_check_aggregate_fail_no_regions(check_aggregate_df):
+    to_tweak = {
+        'model': 'MSG-GLB',
+        'scenario': 'a_scen_2',
+        'region': 'World',
+        'variable': 'Emissions|C2F6|Solvents',
+        'unit': 'kt C2F6/yr',
+    }
+    run_check_agg_fail(check_aggregate_df, to_tweak, 'aggregate')
+
+
+def test_df_check_aggregate_region_fail(check_aggregate_df):
+    to_tweak = {
+        'model': 'IMG',
+        'scenario': 'a_scen_2',
+        'region': 'World',
+        'variable': 'Emissions|CO2',
+        'unit': 'Mt CO2/yr',
+    }
+    run_check_agg_fail(check_aggregate_df, to_tweak, 'region')
+
+
+def test_df_check_aggregate_region_fail_no_subsector(check_aggregate_df):
+    to_tweak = {
+        'model': 'MSG-GLB',
+        'scenario': 'a_scen_2',
+        'region': 'R5REF',
+        'variable': 'Emissions|CH4',
+        'unit': 'Mt CH4/yr',
+    }
+    run_check_agg_fail(check_aggregate_df, to_tweak, 'region')
+
+
+def test_df_check_aggregate_region_fail_world_only_var(check_aggregate_df):
+    to_tweak = {
+        'model': 'MSG-GLB',
+        'scenario': 'a_scen_2',
+        'region': 'World',
+        'variable': 'Emissions|CO2|Agg Agg',
+        'unit': 'Mt CO2/yr',
+    }
+
+    run_check_agg_fail(
+        check_aggregate_df, to_tweak, 'region-world-only-contrib'
+    )
+
+
+def test_df_check_aggregate_regions_errors(check_aggregate_regional_df):
+    # these tests should fail because our dataframe has continents and regions
+    # so checking without providing components leads to double counting and
+    # hence failure
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O', 'World'
+    )
+
+    assert len(obs.columns) == 2
+    assert obs.index.get_values()[0] == (
+        'World', 'AIM', 'cscen', 'Emissions|N2O', 'Mt N/yr'
+    )
+
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O', 'REUROPE'
+    )
+
+    assert len(obs.columns) == 2
+    assert obs.index.get_values()[0] == (
+        'REUROPE', 'AIM', 'cscen', 'Emissions|N2O', 'Mt N/yr'
+    )
+
+
+def test_df_check_aggregate_regions_components(check_aggregate_regional_df):
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O', 'World', components=['REUROPE', 'RASIA']
+    )
+    assert obs is None
+
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O|Solvents', 'World', components=['REUROPE', 'RASIA']
+    )
+    assert obs is None
+
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O', 'REUROPE', components=['Germany', 'UK']
+    )
+    assert obs is None
+
+    obs = check_aggregate_regional_df.check_aggregate_regions(
+        'Emissions|N2O|Transport', 'REUROPE', components=['Germany', 'UK']
+    )
+    assert obs is None