Repository: IAMconsortium/pyam
Pull request #98 (2018-09-25 13:45:51)

adding a statistics module
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- [x] Description in RELEASE_NOTES.md Added

# Description of PR

This PR adds a simple summary-statistics module to `pyam`, which acts as a wrapper for `pd.descibe()`. When initializing an instance of  `pyam.Statistics`, the user defines `groupby` and/or `filters` arguments, which will then be applied to all data added to the statistics instance.

This moves the tedious tasks of applying groupbys, filtering, structuring of data into the module, and formatting of the `describe()` output, so that the workflow can focus on "get some data and describe, get more data and describe, etc.", easily producing a coherent summary-statistics output.

The `pyam.Statistics.summarize()`offers simple selection of mean vs. median, interquartile vs. full  range, and rounding/formatting.

The PR also extends the `pyam.filter_by_meta()` function to accept `model` and `scenario` as filter arguments.

danielhuppmann: @gidden, refactored the `Statistics.describe()` to `add()` as you suggested, made the `reindex()` function behave more like `pd.DataFrame.reindex()`, and some more clean-up.
gidden: ready for review?
danielhuppmann: just resolved the merge conflict, ready for review

Patch:
diff --git a/.stickler.yml b/.stickler.yml
index 79d8b7f..ccca240 100644
--- a/.stickler.yml
+++ b/.stickler.yml
@@ -2,7 +2,7 @@ linters:
   flake8:
     max-line-length: 79
     fixer: false
-    ignore: I002
+    ignore: I002, F403
     # stickler doesn't support 'exclude' for flake8 properly, so we disable it
     # below with files.ignore:
     # https://github.com/markstory/lint-review/issues/184
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 06635ab..9f1d2bb 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,7 +1,9 @@
 
 # Next Release
 
+- (#104)[https://github.com/IAMconsortium/pyam/pull/104] fixes a bug with timeseries utils functions, ensures that years are cast as integers
 - (#101)[https://github.com/IAMconsortium/pyam/pull/101] add function `cross_threshold()` to determine years where a timeseries crosses a given threshold
+- (#98)[https://github.com/IAMconsortium/pyam/pull/98] add a module to compute and format summary statistics for timeseries data (wrapper for `pd.describe()`
 - (#95)[https://github.com/IAMconsortium/pyam/pull/95] add a `scatter()` chart in the plotting library using metadata 
 - (#94)[https://github.com/IAMconsortium/pyam/pull/94] `set_meta()` can take pd.DataFrame (with columns `['model', 'scenario']`) as `index` arg
 - (#93)[https://github.com/IAMconsortium/pyam/pull/93] IamDataFrame can be initilialzed from pd.DataFrame with index
diff --git a/doc/source/Statistics.rst b/doc/source/Statistics.rst
new file mode 100644
index 0000000..5e3099c
--- /dev/null
+++ b/doc/source/Statistics.rst
@@ -0,0 +1,14 @@
+.. currentmodule:: pyam
+
+Class Statistics
+================
+
+This class provides a wrapper for generating descriptive summary statistics
+for timeseries data using various groupbys or filters.
+It uses the `pandas.describe()`_ function internally
+and hides the tedious work of filters, groupbys and merging of dataframes.
+
+.. _`pandas.describe()` : https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.describe.html
+
+.. autoclass:: Statistics
+   :members:
diff --git a/doc/source/index.rst b/doc/source/index.rst
index c0815fe..e1fe007 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -106,6 +106,7 @@ See `this guide`_ for guidelines on NumPy/SciPy Documentation conventions.
    :maxdepth: 2
 
    IamDataFrame
+   Statistics
    pyam_functions
    timeseries
    examples/index
diff --git a/pyam/__init__.py b/pyam/__init__.py
index 35cdf2e..b6c69ed 100644
--- a/pyam/__init__.py
+++ b/pyam/__init__.py
@@ -1,5 +1,6 @@
 from pyam.core import *
 from pyam.utils import *
+from pyam.statistics import *
 from pyam.timeseries import *
 from pyam.logger import *
 from pyam.run_control import *
diff --git a/pyam/core.py b/pyam/core.py
index 25712bb..614c366 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -28,6 +28,7 @@ from pyam.utils import (
     years_match,
     isstr,
     islistable,
+    cast_years_to_int,
     META_IDX,
     YEAR_IDX,
     IAMC_IDX,
@@ -67,6 +68,10 @@ class IamDataFrame(object):
         else:
             self.data = read_files(data, **kwargs)
 
+        # cast year column to `int` if necessary
+        if not self.data.year.dtype == 'int64':
+            self.data.year = cast_years_to_int(self.data.year)
+
         # define a dataframe for categorization and other metadata indicators
         self.meta = self.data[META_IDX].drop_duplicates().set_index(META_IDX)
         self.reset_exclude()
@@ -1074,16 +1079,19 @@ def filter_by_meta(data, df, join_meta=False, **kwargs):
     if not set(META_IDX).issubset(data.index.names + list(data.columns)):
         raise ValueError('missing required index dimensions or columns!')
 
-    meta = pd.DataFrame(df.meta[list(kwargs)].copy())
+    meta = pd.DataFrame(df.meta[list(set(kwargs) - set(META_IDX))].copy())
 
     # filter meta by columns
     keep = np.array([True] * len(meta))
     apply_filter = False
     for col, values in kwargs.items():
-        if values is not None:
-            keep_col = pattern_match(meta[col], values)
-            keep &= keep_col
+        if col in META_IDX and values is not None:
+            _col = meta.index.get_level_values(0 if col is 'model' else 1)
+            keep &= pattern_match(_col, values, has_nan=False)
             apply_filter = True
+        elif values is not None:
+            keep &= pattern_match(meta[col], values)
+        apply_filter |= values is not None
     meta = meta[keep]
 
     # set the data index to META_IDX and apply filtered meta index
diff --git a/pyam/statistics.py b/pyam/statistics.py
new file mode 100644
index 0000000..2216246
--- /dev/null
+++ b/pyam/statistics.py
@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from copy import deepcopy
+import numpy as np
+import pandas as pd
+from pyam import filter_by_meta, META_IDX
+from pyam.utils import isstr, islistable
+
+
+class Statistics(object):
+    """This class provides a wrapper for descriptive statistics
+    of IAMC-style timeseries data.
+
+    Parameters
+    ----------
+    df: pyam.IamDataFrame
+        an IamDataFrame from which to retrieve metadata for grouping, filtering
+    groupby: str or dict
+        a column of `df.meta` to be used for the groupby feature,
+        or a dictionary of `{column: list}`, where `list` is used for ordering
+    filters: list of tuples
+        arguments for filtering and describing, either `((index, dict)` or
+        `((index[0], index[1]), dict)`, when also using `groupby`, index must
+        haev length 2.
+    percentiles: list-like of numbers, optional
+        The percentiles to include in the output of `pandas.describe()`.
+        All should fall between 0 and 1. The default is `[.25, .5, .75]`,
+        which returns the 25th, 50th, and 75th percentiles.
+    """
+    def __init__(self, df, groupby=None, filters=None, rows=False,
+                 percentiles=[0.25, 0.5, 0.75]):
+        self.df = df
+        self.idx_depth = None
+
+        # assing `groupby` settings and check that specifications are valid
+        self.col = None
+        self.groupby = None
+        if isstr(groupby):
+            self.col = groupby
+            self.groupby = {groupby: None}
+        elif isinstance(groupby, dict) and len(groupby) == 1:
+            self.col = list(groupby.keys())[0]
+            self.groupby = groupby
+            self.idx_depth = 2
+        elif groupby is not None:
+            raise ValueError('arg `{}` not valid `groupby`'.format(groupby))
+        if self.col is not None and self.col not in df.meta.columns:
+            raise ValueError('column `{}` not in `df.meta`'.format(self.col))
+
+        # if neither groupby nor filters is given, use filters to describe all
+        # and assume that rows are used
+        if groupby is None and filters is None:
+            self.filters = [('', {})]
+            rows = True
+        else:
+            self.filters = filters if filters is not None else []
+
+        # set lists to sort index and subindex
+        self._idx = [] if self.col is None else [self.col]
+        self._sub_idx = self.groupby[self.col] or self.df[self.col].unique() \
+            if self.col is not None else []
+        self._headers, self._subheaders = ([], [])
+
+        # assing `filters` settings and check that specifications are valid
+        for (idx, _filter) in self.filters:
+            # check that index in tuple is valid
+            if isstr(idx):
+                self._add_to_index(idx)
+            else:
+                if not (isinstance(idx, tuple) and len(idx) == 2 and
+                        isstr(idx[0]) or not isstr(idx[1])):
+                    raise ValueError('`{}` is not a valid index'.format(idx))
+                self._add_to_index(idx[0], idx[1])
+            # check that filters in tuple are valid
+            if not isinstance(_filter, dict):
+                raise ValueError('`{}` is not a valid filter'.format(_filter))
+            elif not (set(_filter) - set(META_IDX)).issubset(df.meta):
+                raise ValueError('column `{}` not in `df.meta`'.format(
+                    set(_filter) - set(META_IDX) - set(df.meta)))
+
+        self.stats = None
+        self.rows = [] if rows else None
+
+        # percentiles for passing to `pandas.describe()`
+        self.percentiles = list(percentiles)
+        self._describe_cols = (['count', 'mean', 'std', 'min'] +
+                               ['{:.0%}'.format(i) for i in self.percentiles] +
+                               ['max'])
+
+    def _add_to_index(self, idx, sub_idx=None):
+        # assign index depth if not set
+        if self.idx_depth is None:
+            self.idx_depth = 1 if sub_idx is None else 2
+        # check that index matches depth
+        if self.groupby is not None and sub_idx is None:
+            msg = 'if `groupby` is used, index `{}` must have format `{}`'
+            raise ValueError(msg.format(idx, '(idx0, idx1)'))
+        if self.idx_depth == 1 and sub_idx is not None:
+            raise ValueError('index depth set to 1, found `({}, {})`'
+                             .format(idx, sub_idx))
+        if self.idx_depth == 2 and sub_idx is None:
+            raise ValueError('index depth set to 2, found `({})`'.format(idx))
+
+        # append to lists for sorting index
+        if idx not in self._idx:
+            self._idx.append(idx)
+        if self.idx_depth == 2 and sub_idx not in self._sub_idx:
+            self._sub_idx.append(sub_idx)
+
+    def _add_to_header(self, header, subheader):
+        if header not in self._headers:
+            self._headers.append(header)
+        if islistable(subheader):
+            for s in subheader:
+                if s not in self._subheaders:
+                    self._subheaders.append(s)
+        elif subheader not in self._subheaders:
+            self._subheaders.append(subheader)
+
+    def add(self, data, header, row=None, subheader=None):
+        """Filter `data` by arguments of this SummaryStats instance,
+        then apply `pd.describe()` and format the statistics
+
+        Parameters
+        ----------
+        data : pd.DataFrame or pd.Series
+            data for which summary statistics should be computed
+        header : str
+            column name for descriptive statistics
+        row : str
+            row name for descriptive statistics
+            (required if `pyam.Statistics(rows=True)`)
+        subheader : str, optional
+            column name (level=1) if data is a unnamed `pd.Series`
+        """
+        # verify validity of specifications
+        if self.rows is not None and row is None:
+            raise ValueError('row specification required')
+        if self.rows is None and row is not None:
+            raise ValueError('row arg illegal for this `Statistics` instance')
+        if isinstance(data, pd.Series):
+            if subheader is not None:
+                data.name = subheader
+            elif data.name is None:
+                msg = '`data` must be named `pd.Series` or provide `subheader`'
+                raise ValueError(msg)
+            data = pd.DataFrame(data)
+
+        if self.rows is not None and row not in self.rows:
+            self.rows.append(row)
+
+        _stats = None
+
+        # describe with groupby feature
+        if self.groupby is not None:
+            filter_args = dict(data=data, df=self.df, join_meta=True)
+            filter_args.update(self.groupby)
+            _stats = (
+                filter_by_meta(**filter_args).groupby(self.col)
+                .describe(percentiles=self.percentiles)
+            )
+            _stats = pd.concat([_stats], keys=[self.col], names=[''], axis=0)
+            if self.rows:
+                _stats['row'] = row
+                _stats.set_index('row', append=True, inplace=True)
+            _stats.index.names = [''] * 3 if self.rows else [''] * 2
+
+        # describe with filter feature
+        for (idx, _filter) in self.filters:
+            filter_args = dict(data=data, df=self.df)
+            filter_args.update(_filter)
+            _stats_f = (
+                filter_by_meta(**filter_args)
+                .describe(percentiles=self.percentiles)
+            )
+            _stats_f = pd.DataFrame(_stats_f.unstack()).T
+            if self.idx_depth == 1:
+                levels = [[idx]]
+            else:
+                levels = [[idx[0]], [idx[1]]]
+            lvls, lbls = (levels, [[0]] * self.idx_depth) if not self.rows \
+                else (levels + [[row]], [[0]] * (self.idx_depth + 1))
+            _stats_f.index = pd.MultiIndex(levels=lvls, labels=lbls)
+            _stats = _stats_f if _stats is None else _stats.append(_stats_f)
+
+        # add header
+        _stats = pd.concat([_stats], keys=[header], names=[''], axis=1)
+        subheader = _stats.columns.get_level_values(1).unique()
+        self._add_to_header(header, subheader)
+
+        # set statistics
+        if self.stats is None:
+            self.stats = _stats
+        else:
+            self.stats = _stats.combine_first(self.stats)
+
+    def reindex(self, copy=True):
+        """Reindex the summary statistics dataframe"""
+        ret = deepcopy(self) if copy else self
+
+        ret.stats = ret.stats.reindex(index=ret._idx, level=0)
+        if ret.idx_depth == 2:
+            ret.stats = ret.stats.reindex(index=ret._sub_idx, level=1)
+        if ret.rows is not None:
+            ret.stats = ret.stats.reindex(index=ret.rows, level=ret.idx_depth)
+
+        ret.stats = ret.stats.reindex(columns=ret._headers, level=0)
+        ret.stats = ret.stats.reindex(columns=ret._subheaders, level=1)
+        ret.stats = ret.stats.reindex(columns=ret._describe_cols, level=2)
+
+        if copy:
+            return ret
+
+    def summarize(self, center='mean', fullrange=None, interquartile=None,
+                  custom_format='{:.2f}'):
+        """Format the compiled statistics to a concise string output
+
+        Parameter
+        ---------
+        center : str, default `mean`
+            what to return as 'center' of the summary: `mean`, `50%`, `median`
+        fullrange : bool, default None
+            return full range of data if True or `fullrange`, `interquartile`
+            and `format_spec` are None
+        interquartile : bool, default None
+            return interquartile range if True
+        custom_format : formatting specifications
+        """
+        # call `reindex()` to reorder index and columns
+        self.reindex(copy=False)
+
+        center = 'median' if center == '50%' else center
+        if fullrange is None and interquartile is None:
+            fullrange = True
+        return self.stats.apply(format_rows, center=center,
+                                fullrange=fullrange,
+                                interquartile=interquartile,
+                                custom_format=custom_format,
+                                axis=1, raw=False)
+
+# %% auxiliary functions
+
+
+def format_rows(row, center, fullrange=None, interquartile=None,
+                custom_format='{:.2f}'):
+    """Format a row with `describe()` columns to a concise string"""
+    if (fullrange or 0) + (interquartile or 0) == 1:
+        legend = '{} ({})'.format(center, 'max, min' if fullrange is True
+                                  else 'interquartile range')
+        index = row.index.droplevel(2).drop_duplicates()
+        count_arg = dict(tuples=[('count', '')], names=[None, legend])
+    else:
+        msg = 'displaying multiple range formats simultaneously not supported'
+        raise NotImplementedError(msg)
+
+    ret = pd.Series(index=pd.MultiIndex.from_tuples(**count_arg).append(index))
+
+    row = row.sort_index()
+    center = '50%' if center == 'median' else center
+
+    # get maximum of `count` and write to first entry of return series
+    count = max([i for i in row.loc[(slice(None), slice(None), 'count')]
+                 if not np.isnan(i)])
+    ret.loc[('count', '')] = ('{:.0f}'.format(count)) if count > 1 else ''
+
+    # set upper and lower for the range
+    upper, lower = ('max', 'min') if fullrange is True else ('75%', '25%')
+
+    # format `describe()` columns to string output
+    for i in index:
+        x = row.loc[i]
+        _count = x['count']
+        if np.isnan(_count) or _count == 0:
+            s = ''
+        elif _count > 1:
+            s = '{f} ({f}, {f})'.format(f=custom_format)\
+                .format(x[center], x[upper], x[lower])
+        elif _count == 1:
+            s = '{f}'.format(f=custom_format).format(x['50%'])
+        # add count of this section as `[]` if different from count_max
+        if 0 < _count < count:
+            s += ' [{:.0f}]'.format(_count)
+        ret.loc[i] = s
+
+    return ret
diff --git a/pyam/timeseries.py b/pyam/timeseries.py
index f150f88..c4c932f 100644
--- a/pyam/timeseries.py
+++ b/pyam/timeseries.py
@@ -1,14 +1,12 @@
 # -*- coding: utf-8 -*-
 
 import numpy as np
-import warnings
-
-from pyam.utils import (
-    isstr
-)
+from pyam.logger import logger
+from pyam.utils import isstr, cast_years_to_int
 
 # %%
 
+
 def fill_series(x, year):
     """Returns the value of a timeseries (indexed over years) for a year
     by linear interpolation.
@@ -50,19 +48,21 @@ def cumulative(x, first_year, last_year):
     last_year: int
         last year of the sum (inclusive)
     """
-
+    # if the timeseries does not cover the range `[first_year, last_year]`,
+    # return nan to avoid erroneous aggregation
     if min(x.index) > first_year:
-        # if the timeseries does not cover the range `[first_year, last_year]`,
-        # replace by nan to avoid erroneous aggregation
-        warnings.warn('timeseries {} does not start by {}'.format(x.name,
-                      first_year))
+        logger().warning('the timeseries `{}` does not start by {}'.format(
+            x.name or x, first_year))
         return np.nan
-
     if max(x.index) < last_year:
-        warnings.warn('the timeseries {} does not extend until {}'
-                      .format(x.name, last_year))
+        logger().warning('the timeseries `{}` does not extend until {}'
+                         .format(x.name or x, last_year))
         return np.nan
 
+    # cast tiemseries colums to `int` if necessary
+    if not x.index.dtype == 'int64':
+        cast_years_to_int(x, index=True)
+
     x[first_year] = fill_series(x, first_year)
     x[last_year] = fill_series(x, last_year)
 
diff --git a/pyam/utils.py b/pyam/utils.py
index f678255..9fd69e1 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -228,7 +228,7 @@ def find_depth(data, s, level):
     return list(map(apply_test, data))
 
 
-def pattern_match(data, values, level=None, regexp=False):
+def pattern_match(data, values, level=None, regexp=False, has_nan=True):
     """
     matching of model/scenario names, variables, regions, and meta columns to
     pseudo-regex (if `regexp == False`) for filtering (str, int, bool)
@@ -239,7 +239,8 @@ def pattern_match(data, values, level=None, regexp=False):
 
     # issue (#40) with string-to-nan comparison, replace nan by empty string
     _data = data.copy()
-    _data.loc[[np.isnan(i) if not isstr(i) else False for i in _data]] = ''
+    if has_nan:
+        _data.loc[[np.isnan(i) if not isstr(i) else False for i in _data]] = ''
 
     for s in values:
         if isstr(s):
@@ -268,3 +269,20 @@ def years_match(data, years):
     """
     years = [years] if isinstance(years, int) else years
     return data.isin(years)
+
+
+def cast_years_to_int(x, index=False):
+    """Formatting series or timeseries columns to int and checking validity.
+    If `index=False`, the function works on the `pd.Series x`; else,
+    the function casts the index of `x` to int and returns x with a new index.
+    """
+    _x = x.index if index else x
+    cols = list(map(int, _x))
+    error = _x[cols != _x]
+    if not error.empty:
+        raise ValueError('invalid values `{}`'.format(list(error)))
+    if index:
+        x.index = cols
+        return x
+    else:
+        return _x


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 47ca9ff..8c0e2ce 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -33,6 +33,17 @@ def test_init_df_with_index(test_pd_df):
     pd.testing.assert_frame_equal(df.timeseries().reset_index(), test_pd_df)
 
 
+def test_init_df_with_float_cols_raises(test_pd_df):
+    _test_df = test_pd_df.rename(columns={2005: 2005.5, 2010: 2010.})
+    pytest.raises(ValueError, IamDataFrame, data=_test_df)
+
+
+def test_init_df_with_float_cols(test_pd_df):
+    _test_df = test_pd_df.rename(columns={2005: 2005., 2010: 2010.})
+    obs = IamDataFrame(_test_df).timeseries().reset_index()
+    pd.testing.assert_series_equal(obs[2005], test_pd_df[2005])
+
+
 def test_init_df_from_timeseries(test_df):
     df = IamDataFrame(test_df.timeseries())
     pd.testing.assert_frame_equal(df.timeseries(), test_df.timeseries())
diff --git a/tests/test_statistics.py b/tests/test_statistics.py
new file mode 100644
index 0000000..e8e076c
--- /dev/null
+++ b/tests/test_statistics.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+import pytest
+import pandas as pd
+from pyam import Statistics
+
+
+def stats_add(stats, plot_df):
+    # test describing as pd.DataFrame
+    primary = plot_df.filter(variable='Primary Energy', year=2005).timeseries()
+    stats.add(data=primary, header='primary')
+    # test describing as unamed pd.Series with `subheader` arg
+    coal = plot_df.filter(variable='Primary Energy|Coal').timeseries()[2010]
+    coal.name = None
+    stats.add(data=coal, header='coal', subheader=2005)
+
+    # repeat `add()` to ensure that previous data is overwritten
+    coal = plot_df.filter(variable='Primary Energy|Coal').timeseries()[2005]
+    coal.name = None
+    stats.add(data=coal, header='coal', subheader=2005)
+    return stats
+
+
+def stats_add_with_rows(stats, plot_df):
+    # test describing as pd.DataFrame
+    primary = plot_df.filter(variable='Primary Energy', year=2005).timeseries()
+    stats.add(data=primary, header='primary', row='first')
+    # test describing as unamed pd.Series with `subheader` arg
+    coal = plot_df.filter(variable='Primary Energy|Coal').timeseries()[2010]
+    coal.name = None
+    stats.add(data=coal, header='coal', subheader=2005, row='another')
+    # repeat `add()` to ensure that previous data is overwritten
+    coal = plot_df.filter(variable='Primary Energy|Coal').timeseries()[2005]
+    coal.name = None
+    stats.add(data=coal, header='coal', subheader=2005, row='another')
+    return stats
+
+
+def test_statistics(plot_df):
+    plot_df.set_meta(meta=['a', 'b', 'b', 'a'], name='category')
+    stats = Statistics(df=plot_df,
+                       groupby={'category': ['b', 'a']},
+                       filters=[(('scen', 'test'),
+                                 {'scenario': 'test_scenario'})])
+    obs = stats_add(stats, plot_df).summarize(custom_format='{:.0f}')
+
+    idx = pd.MultiIndex(levels=[['category', 'scen'], ['b', 'a', 'test']],
+                        labels=[[0, 0, 1], [0, 1, 2]], names=['', ''])
+    cols = pd.MultiIndex(levels=[['count', 'primary', 'coal'], ['', 2005]],
+                         labels=[[0, 1, 2], [0, 1, 1]],
+                         names=[None, 'mean (max, min)'])
+    exp = pd.DataFrame(data=[['2', '1 (2, 1)', '0 (0, 0)'],
+                             ['2', '1 (1, 1)', '0 (0, 0)'],
+                             ['2', '1 (1, 1)', '0 (0, 0)']],
+                       index=idx, columns=cols)
+    pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_statistics_mismatching_groupby_index(plot_df):
+    pytest.raises(ValueError, Statistics, df=plot_df,
+                  groupby={'category': ['b', 'a']},
+                  filters=[(('test'), {'scenario': 'test_scenario'})])
+
+
+def test_statistics_mismatching_filters_depth(plot_df):
+    pytest.raises(ValueError, Statistics, df=plot_df,
+                  groupby={'category': ['b', 'a']},
+                  filters=[(('test'), {'scenario': 'test_scenario'}),
+                           (('test', 'test'), {'scenario': 'test_scenario'})])
+
+
+def test_statistics_by_filter(plot_df):
+    stats = Statistics(df=plot_df,
+                       filters=[('test', {'scenario': 'test_scenario'})])
+    obs = stats_add(stats, plot_df).summarize(interquartile=True)
+
+    idx = pd.MultiIndex(levels=[['test']], labels=[[0]])
+    cols = pd.MultiIndex(levels=[['count', 'primary', 'coal'],
+                                 ['', 2005]],
+                         labels=[[0, 1, 2], [0, 1, 1]],
+                         names=[None, 'mean (interquartile range)'])
+    exp = pd.DataFrame(data=['2', '0.85 (0.93, 0.77)', '0.42 (0.46, 0.39)'],
+                       index=cols, columns=idx).T
+    pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_statistics_with_row_required(plot_df):
+    stats = Statistics(df=plot_df)
+    primary = plot_df.filter(variable='Primary Energy', year=2005).timeseries()
+    # test that kwarg `row` is required for `add`
+    pytest.raises(ValueError, stats.add, data=primary, header='primary')
+
+
+def test_statistics_with_rows(plot_df):
+    stats = Statistics(df=plot_df,
+                       filters=[('test', {'scenario': 'test_scenario'})],
+                       rows=True)
+    obs = stats_add_with_rows(stats, plot_df).summarize(center='50%')
+
+    idx = pd.MultiIndex(levels=[['test'], ['first', 'another']],
+                        labels=[[0, 0], [0, 1]])
+    cols = pd.MultiIndex(levels=[['count', 'primary', 'coal'], ['', 2005]],
+                         labels=[[0, 1, 2], [0, 1, 1]],
+                         names=[None, 'median (max, min)'])
+    exp = pd.DataFrame(data=[['2', '0.85 (1.00, 0.70)', ''],
+                             ['2', '', '0.42 (0.50, 0.35)']],
+                       index=idx, columns=cols)
+    pd.testing.assert_frame_equal(obs, exp)
+
+
+def test_statistics_with_percentiles(plot_df):
+    stats = Statistics(df=plot_df,
+                       filters=[('test', {'scenario': 'test_scenario'})],
+                       percentiles=[0.05, 0.95])
+    stats = stats_add(stats, plot_df)
+    obs = set(stats.stats.columns.get_level_values(2))
+    assert obs == set(['count', 'mean', 'std', 'min', '5%', '50%', '95%',
+                       'max'])
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index ef60967..d3eaac9 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -3,10 +3,42 @@
 
 import numpy as np
 import pandas as pd
-from pyam import cross_threshold
+from pyam.logger import logger
+from pyam import fill_series, cumulative, cross_threshold, cast_years_to_int
 import pytest
 
 
+def test_fill_series():
+    # note that the series is not order and the index is defined as float
+    y = pd.Series(data=[np.nan, 1, 4, 1], index=[2002., 2008., 2005., 2013.])
+    assert fill_series(y, 2006) == 3.
+
+
+def test_fill_series_out_of_range():
+    y = pd.Series(data=[np.nan, 1, 3, 1], index=[2002., 2005., 2007., 2013.])
+    assert fill_series(y, 2001) is np.nan
+
+
+def test_cols_to_int():
+    y = pd.Series(data=[np.nan, 1, 3, 1], index=[2002., 2007.5, 2003., 2013.])
+    pytest.raises(ValueError, cast_years_to_int, x=y)
+
+
+def test_cumulative():
+    # note that the series is not order and the index is defined as float
+    y = pd.Series(data=[np.nan, 1, 3, 1], index=[2002., 2007., 2003., 2013.])
+    assert cumulative(y, 2008, 2013) == 6
+
+
+def test_cumulative_out_of_range():
+    # set logger level to exclude warnings in unit test output
+    logger().setLevel('ERROR')
+    # note that the series is not order and the index is defined as float
+    y = pd.Series(data=[np.nan, 1, 3, 1], index=[2002., 2005., 2007., 2013.])
+    assert cumulative(y, 2008, 2015) is np.nan
+    logger().setLevel('NOTSET')
+
+
 def test_cross_treshold():
     y = pd.Series(data=[np.nan, 1, 3, 1], index=[2002, 2005, 2007, 2013])
     obs = cross_threshold(y, 2)