Repository: IAMconsortium/pyam
Pull request #52 (2018-03-19 14:43:36)

check aggregate
This PR adds a function to check whether the sub-categories of a timeseries (e.g., `foo|bar1`, `foo|bar2`) match the values of the variable of the level above (e.g., `foo`). This helps to identify reporting errors and missing entries.

gidden: Some small changes requested. I think this is also a feature that would be very useful to document with examples (especially because I am not immediately sure how to use it). Perhaps it is worth starting an "Examples" page or something similar in the documentation to show how to use this function? I would imagine we will add lots more too, basically like the current plotting gallery but for other operations.
danielhuppmann: @gidden, thanks for the feedback! should be good to go...
danielhuppmann: thanks @gidden for the suggestion with `np.isclose()` - a lot nice!also incorporated your renaming suggestions

rebased on top of PR #55, ready for review

gidden: This might be easiest to cherry pick now

```
git checkout master
git checkout -b new-branch
git cherry-pick 26d63e0..bc41dd1
```
danielhuppmann: rebased on `2695bc`
danielhuppmann: should be good to be merged, @gidden 
gidden: lgtm!

Patch:
diff --git a/pyam/core.py b/pyam/core.py
index e3f7b8c..0755fcb 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -29,6 +29,7 @@ from pyam.utils import (
     isstr,
     islistable,
     META_IDX,
+    YEAR_IDX,
     IAMC_IDX,
     SORT_IDX,
     LONG_IDX,
@@ -416,17 +417,12 @@ class IamDataFrame(object):
         """
         df = _apply_criteria(self.data, criteria, in_range=False)
 
-        if exclude_on_fail:
-            idx = _meta_idx(df)
-            self.meta.loc[idx, 'exclude'] = True
-
         if not df.empty:
             msg = '{} of {} data points to not satisfy the criteria'
             logger().info(msg.format(len(df), len(self.data)))
 
-            if exclude_on_fail and len(idx) > 0:
-                logger().info('{} non-valid scenario{} will be excluded'
-                              .format(len(idx), '' if len(idx) == 1 else 's'))
+            if exclude_on_fail and len(df) > 0:
+                self._exclude_on_fail(df)
 
             return df
 
@@ -474,6 +470,56 @@ class IamDataFrame(object):
         if not inplace:
             return ret
 
+    def check_aggregate(self, variable, components=None, units=None,
+                        exclude_on_fail=False, multiplier=1, **kwargs):
+        """Check whether the timeseries data match the aggregation
+        of components or sub-categories
+
+        Parameters
+        ----------
+        variable: str
+            variable to be checked for matching aggregation of sub-categories
+        components: list of str, default None
+            list of variables, defaults to all sub-categories of `variable`
+        units: str or list of str, default None
+            filter variable and components for given unit(s)
+        exclude_on_fail: boolean, default False
+            flag scenarios failing validation as `exclude: True`
+        multiplier: number, default 1
+            factor when comparing variable and sum of components
+        kwargs: passed to `np.isclose()`
+        """
+        # default components to all variables one level below `variable`
+        if components is None:
+            components = self.filter(variable='{}|*'.format(variable),
+                                     level=0).variables()
+
+        # filter and groupby data, use `pd.Series.align` for machting index
+        df_variable, df_components = (
+                _aggregate_by_variables(self.data, variable, units)
+                .align(_aggregate_by_variables(self.data, components, units))
+                )
+
+        # use `np.isclose` for checking match
+        diff = df_variable[~np.isclose(df_variable, multiplier * df_components,
+                                       **kwargs)]
+
+        if len(diff):
+            msg = '{} of {} data points are not aggregates of components'
+            logger().info(msg.format(len(diff), len(df_variable)))
+
+            if exclude_on_fail:
+                self._exclude_on_fail(diff.index.droplevel([2, 3]))
+
+            return diff.unstack().rename_axis(None, axis=1)
+
+    def _exclude_on_fail(self, df):
+        """Assign a selection of scenarios as `exclude: True` in meta"""
+        idx = df if isinstance(df, pd.MultiIndex) else _meta_idx(df)
+        self.meta.loc[idx, 'exclude'] = True
+        logger().info('{} non-valid scenario{} will be excluded'
+                      .format(len(idx), '' if len(idx) == 1 else 's'))
+
     def filter(self, filters=None, keep=True, inplace=False, **kwargs):
         """Return a filtered IamDataFrame (i.e., a subset of current data)
 
@@ -736,6 +782,17 @@ def _meta_idx(data):
     return data[META_IDX].drop_duplicates().set_index(META_IDX).index
 
 
+def _aggregate_by_variables(df, variables, units=None):
+    variables = [variables] if isstr(variables) else variables
+    df = df[df.variable.isin(variables)]
+
+    if units is not None:
+        units = [units] if isstr(units) else units
+        df = df[df.unit.isin(units)]
+
+    return df.groupby(YEAR_IDX).sum()['value']
+
+
 def _apply_filters(data, meta, filters):
     keep = np.array([True] * len(data))
 
@@ -830,7 +887,8 @@ def validate(df, criteria={}, exclude_on_fail=False, **kwargs):
     Parameters
     ----------
     df: IamDataFrame instance
-    args and kwargs: see IamDataFrame.validate() and .filter() for details
+    args: see `IamDataFrame.validate()` for details
+    kwargs: passed to `df.filter()`
     """
     fdf = df.filter(**kwargs)
     if len(fdf.data) > 0:
@@ -846,7 +904,8 @@ def require_variable(df, variable, unit=None, year=None, exclude_on_fail=False,
     Parameters
     ----------
     df: IamDataFrame instance
-    args and kwargs: see IamDataFrame.validate() and .filter() for details
+    args: see `IamDataFrame.require_variable()` for details
+    kwargs: passed to `df.filter()`
     """
     fdf = df.filter(**kwargs)
     if len(fdf.data) > 0:
@@ -864,7 +923,8 @@ def categorize(df, name, value, criteria,
     Parameters
     ----------
     df: IamDataFrame instance
-    args and kwargs: see IamDataFrame.categorize()  and .filter() for details
+    args: see `IamDataFrame.categorize()` for details
+    kwargs: passed to `df.filter()`
     """
     fdf = df.filter(**kwargs)
     fdf.categorize(name=name, value=value, criteria=criteria, color=color,
@@ -875,3 +935,23 @@ def categorize(df, name, value, criteria,
         df.meta[name].update(fdf.meta[name])
     else:
         df.meta[name] = fdf.meta[name]
+
+
+def check_aggregate(df, variable, components=None, units=None,
+                    exclude_on_fail=False, multiplier=1, **kwargs):
+    """Check whether the timeseries values match the aggregation
+    of sub-categories
+
+    Parameters
+    ----------
+    df: IamDataFrame instance
+    args: see IamDataFrame.check_aggregate() for details
+    kwargs: passed to `df.filter()`
+    """
+    fdf = df.filter(**kwargs)
+    if len(fdf.data) > 0:
+        vdf = fdf.check_aggregate(variable=variable, components=components,
+                                  units=units, exclude_on_fail=exclude_on_fail,
+                                  multiplier=multiplier)
+        df.meta['exclude'] |= fdf.meta['exclude']  # update if any excluded
+        return vdf
diff --git a/pyam/utils.py b/pyam/utils.py
index abf7618..f6f2758 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -28,6 +28,7 @@ from pyam.logger import logger
 
 # common indicies
 META_IDX = ['model', 'scenario']
+YEAR_IDX = ['model', 'scenario', 'region', 'year']
 IAMC_IDX = ['model', 'scenario', 'region', 'variable', 'unit']
 SORT_IDX = ['model', 'scenario', 'variable', 'year', 'region']
 LONG_IDX = IAMC_IDX + ['year']


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 4ed6993..9ffe0b9 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -7,7 +7,7 @@ import pandas as pd
 from numpy import testing as npt
 
 from pyam import IamDataFrame, plotting, validate, categorize, \
-    require_variable
+    require_variable, check_aggregate
 from pyam.core import _meta_idx
 
 
@@ -225,6 +225,28 @@ def test_validate_top_level(meta_df):
     assert list(meta_df['exclude']) == [False, True]
 
 
+def test_check_aggregate_fail(meta_df):
+    obs = meta_df.check_aggregate('Primary Energy', exclude_on_fail=True)
+    assert len(obs.columns) == 2
+    assert obs.index.get_values()[0] == ('a_model', 'a_scenario', 'World')
+
+
+def test_check_aggregate_pass(meta_df):
+    # make data complete so that aggregtion-test passes
+    df = pd.DataFrame({'model': 'a_model', 'scenario': 'a_scenario',
+                       'region': 'World', 'variable': 'Primary Energy|Gas',
+                       'unit': 'EJ/y', 'year': [2005, 2010], 'value': [.5, 3]})
+    meta_df.data = meta_df.data.append(df, ignore_index=True)
+    obs = meta_df.filter(scenario='a_scenario').check_aggregate('Primary Energy')
+    assert obs is None
+
+
+def test_check_aggregate_top_level(meta_df):
+    obs = check_aggregate(meta_df, variable='Primary Energy', year=2005)
+    assert len(obs.columns) == 1
+    assert obs.index.get_values()[0] == ('a_model', 'a_scenario', 'World')
+
+
 def test_category_none(meta_df):
     meta_df.categorize('category', 'Testing', {'Primary Energy': {'up': 0.8}})
     assert 'category' not in meta_df.meta.columns