Repository: IAMconsortium/pyam

Issue:
Check output consistency of `.validate` and similar
Came across an issue with the newest version where an output of `.validate()`, which I think was previously a `pd.DataFrame` is now a `pd.Series`.
Not a problem in itself - but scripts that then take this output and perform operations that only work with DataFrame will throw an error. - So backward compatibility is reduced.

In our case it was `.drop(..., axis=1)`  (i.e. drop columns), which is not possible on  `pd.Series`

Error given is:
`ValueError: No axis named 1 for object type <class 'pandas.core.series.Series'>`


Discussion:
byersiiasa: will likely test this on Monday/Tuesday as will run soem vetting scripts then
danielhuppmann: merging to collect clean-up PRs before the next release, please start a new issue referencing this and #428 if problems are not solved

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 6322dee..5ba5190 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -13,6 +13,7 @@ via getter and setter functions.
 
 - [#437](https://github.com/IAMconsortium/pyam/pull/437) Improved test for appending mismatched timeseries
 - [#436](https://github.com/IAMconsortium/pyam/pull/436) Raise an error with appending mismatching timeseries index dimensions
+- [#429](https://github.com/IAMconsortium/pyam/pull/429) Fix return type of `validate()` after data refactoring
 - [#427](https://github.com/IAMconsortium/pyam/pull/427) Add an `info()` function and use in `print(IamDataFrame)`
 - [#424](https://github.com/IAMconsortium/pyam/pull/424) Add a tutorial reading results from a GAMS model (via a gdx file).
 - [#420](https://github.com/IAMconsortium/pyam/pull/420) Add a `_data` object (implemented as a pandas.Series) to handle timeseries data internally.
diff --git a/pyam/core.py b/pyam/core.py
index 4f05b01..7136f6e 100755
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -737,6 +737,13 @@ class IamDataFrame(object):
             ('up' and 'lo' for respective bounds, 'year' for years)
         exclude_on_fail : bool, optional
             flag scenarios failing validation as `exclude: True`
+
+        Returns
+        -------
+        :class:`pandas.DataFrame`
+            All data points that do not satisfy the criteria.
+        None
+            If all scenarios satisfy the criteria.
         """
         df = _apply_criteria(self._data, criteria, in_range=False)
 
@@ -746,7 +753,7 @@ class IamDataFrame(object):
 
             if exclude_on_fail and len(df) > 0:
                 self._exclude_on_fail(df)
-            return df
+            return df.reset_index()
 
     def rename(self, mapping=None, inplace=False, append=False,
                check_duplicates=True, **kwargs):


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 13c2950..023e181 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -7,14 +7,12 @@ import numpy as np
 import pandas as pd
 from numpy import testing as npt
 
-from pyam import IamDataFrame, validate, categorize, \
-    require_variable, filter_by_meta, META_IDX, IAMC_IDX, sort_data, compare
+from pyam import IamDataFrame, filter_by_meta, META_IDX, IAMC_IDX, sort_data,\
+    compare
 from pyam.core import _meta_idx, concat
 from pyam.utils import isstr
 from pyam.testing import assert_iamframe_equal
 
-from conftest import TEST_DTS
-
 
 df_filter_by_meta_matching_idx = pd.DataFrame([
     ['model_a', 'scen_a', 'region_1', 1],
@@ -566,169 +564,6 @@ def test_meta_idx(test_df):
     assert len(_meta_idx(test_df.data)) == 2
 
 
-def test_require_variable(test_df):
-    obs = test_df.require_variable(variable='Primary Energy|Coal',
-                                   exclude_on_fail=True)
-    assert len(obs) == 1
-    assert obs.loc[0, 'scenario'] == 'scen_b'
-
-    assert list(test_df['exclude']) == [False, True]
-
-
-def test_require_variable_top_level(test_df):
-    obs = require_variable(test_df, variable='Primary Energy|Coal',
-                           exclude_on_fail=True)
-    assert len(obs) == 1
-    assert obs.loc[0, 'scenario'] == 'scen_b'
-
-    assert list(test_df['exclude']) == [False, True]
-
-
-def test_require_variable_year_list(test_df):
-    years = [2005, 2010]
-
-    # checking for variables that have ANY of the years in the list
-    df = IamDataFrame(test_df.data[1:])
-    df.require_variable(variable='Primary Energy',
-                        year=years,
-                        exclude_on_fail=True)
-    df.filter(exclude=False, inplace=True)
-
-    assert len(df.variables()) == 2
-    assert len(df.scenarios()) == 2
-
-    # checking for variables that have ALL of the years in the list
-    df = IamDataFrame(test_df.data[1:])
-    for y in years:
-        df.require_variable(variable='Primary Energy',
-                            year=y,
-                            exclude_on_fail=True)
-    df.filter(exclude=False, inplace=True)
-
-    assert len(df.variables()) == 1
-    assert len(df.scenarios()) == 1
-
-
-def test_validate_all_pass(test_df):
-    obs = test_df.validate(
-        {'Primary Energy': {'up': 10}}, exclude_on_fail=True)
-    assert obs is None
-    assert len(test_df.data) == 6  # data unchanged
-
-    assert list(test_df['exclude']) == [False, False]  # none excluded
-
-
-def test_validate_nonexisting(test_df):
-    obs = test_df.validate({'Primary Energy|Coal': {'up': 2}},
-                           exclude_on_fail=True)
-    assert len(obs) == 1
-    assert obs.index.get_level_values('scenario').values[0] == 'scen_a'
-
-    assert list(test_df['exclude']) == [True, False]  # scenario with failed
-    # validation excluded, scenario with non-defined value passes validation
-
-
-def test_validate_up(test_df):
-    obs = test_df.validate({'Primary Energy': {'up': 6.5}},
-                           exclude_on_fail=False)
-    assert len(obs) == 1
-    if 'year' in test_df.data:
-        assert obs.index.get_level_values('year').values[0] == 2010
-    else:
-        exp_time = pd.to_datetime(datetime.datetime(2010, 7, 21))
-        assert pd.to_datetime(obs.index.get_level_values('time')
-                              .values[0]).date() == exp_time
-
-    assert list(test_df['exclude']) == [False, False]  # assert none excluded
-
-
-def test_validate_lo(test_df):
-    obs = test_df.validate({'Primary Energy': {'up': 8, 'lo': 2.0}})
-    assert len(obs) == 1
-    if 'year' in test_df.data:
-        assert obs.index.get_level_values('year').values[0] == 2005
-    else:
-        exp_year = pd.to_datetime(datetime.datetime(2005, 6, 17))
-        assert pd.to_datetime(obs.index.get_level_values('time')
-                              .values[0]).date() == exp_year
-
-    assert list(obs.index.get_level_values('scenario').values) == ['scen_a']
-
-
-def test_validate_both(test_df):
-    obs = test_df.validate({'Primary Energy': {'up': 6.5, 'lo': 2.0}})
-    assert len(obs) == 2
-    if 'year' in test_df.data:
-        assert list(obs.index.get_level_values('year').values) == [2005, 2010]
-    else:
-        exp_time = pd.to_datetime(TEST_DTS)
-        obs.index.set_levels(obs.index.get_level_values('time')
-                                .normalize(), level=5)
-        assert (pd.to_datetime(obs.index.get_level_values('time').values)
-                .date == exp_time).all()
-
-    exp = ['scen_a', 'scen_b']
-    assert list(obs.index.get_level_values('scenario').values) == exp
-
-
-def test_validate_year(test_df):
-    obs = test_df.validate({'Primary Energy': {'up': 5.0, 'year': 2005}},
-                           exclude_on_fail=False)
-    assert obs is None
-
-    obs = test_df.validate({'Primary Energy': {'up': 5.0, 'year': 2010}},
-                           exclude_on_fail=False)
-    assert len(obs) == 2
-
-
-def test_validate_exclude(test_df):
-    test_df.validate({'Primary Energy': {'up': 6.0}}, exclude_on_fail=True)
-    assert list(test_df['exclude']) == [False, True]
-
-
-def test_validate_top_level(test_df):
-    obs = validate(test_df, criteria={'Primary Energy': {'up': 6.0}},
-                   exclude_on_fail=True, variable='Primary Energy')
-    assert len(obs) == 1
-    if 'year' in test_df._data.index.names:
-        assert obs.index.get_level_values('year').values[0] == 2010
-    else:
-        exp_time = datetime.datetime(2010, 7, 21).date()
-        obs_time = obs.index.get_level_values('time')[0].date()
-        assert exp_time == obs_time
-    assert list(test_df['exclude']) == [False, True]
-
-
-def test_category_none(test_df):
-    test_df.categorize('category', 'Testing', {'Primary Energy': {'up': 0.8}})
-    assert 'category' not in test_df.meta.columns
-
-
-def test_category_pass(test_df):
-    dct = {'model': ['model_a', 'model_a'],
-           'scenario': ['scen_a', 'scen_b'],
-           'category': ['foo', None]}
-    exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
-
-    test_df.categorize('category', 'foo', {'Primary Energy':
-                                           {'up': 6, 'year': 2010}})
-    obs = test_df['category']
-    pd.testing.assert_series_equal(obs, exp)
-
-
-def test_category_top_level(test_df):
-    dct = {'model': ['model_a', 'model_a'],
-           'scenario': ['scen_a', 'scen_b'],
-           'category': ['foo', None]}
-    exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
-
-    categorize(test_df, 'category', 'foo',
-               criteria={'Primary Energy': {'up': 6, 'year': 2010}},
-               variable='Primary Energy')
-    obs = test_df['category']
-    pd.testing.assert_series_equal(obs, exp)
-
-
 def test_interpolate(test_pd_df):
     _df = test_pd_df.copy()
     _df['foo'] = ['bar', 'baz', 2]  # add extra_col (check for #351)
diff --git a/tests/test_feature_validation.py b/tests/test_feature_validation.py
new file mode 100644
index 0000000..d7698c0
--- /dev/null
+++ b/tests/test_feature_validation.py
@@ -0,0 +1,166 @@
+import pandas as pd
+import pandas.testing as pdt
+from pyam import IamDataFrame, validate, categorize, require_variable, META_IDX
+
+
+def test_require_variable_pass(test_df):
+    # checking that the return-type is correct
+    obs = test_df.require_variable(variable='Primary Energy',
+                                   exclude_on_fail=True)
+    assert obs is None
+    assert list(test_df['exclude']) == [False, False]
+
+
+def test_require_variable(test_df):
+    exp = pd.DataFrame([['model_a', 'scen_b']], columns=META_IDX)
+
+    # checking that the return-type is correct
+    obs = test_df.require_variable(variable='Primary Energy|Coal')
+    pdt.assert_frame_equal(obs, exp)
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = test_df.require_variable(variable='Primary Energy|Coal',
+                                   exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, exp)
+    assert list(test_df['exclude']) == [False, True]
+
+
+def test_require_variable_top_level(test_df):
+    exp = pd.DataFrame([['model_a', 'scen_b']], columns=META_IDX)
+
+    # checking that the return-type is correct
+    obs = require_variable(test_df, variable='Primary Energy|Coal')
+    pdt.assert_frame_equal(obs, exp)
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = require_variable(test_df, variable='Primary Energy|Coal',
+                           exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, exp)
+    assert list(test_df['exclude']) == [False, True]
+
+
+def test_require_variable_year_list(test_df):
+    # drop first data point
+    df = IamDataFrame(test_df.data[1:])
+    # checking for variables that have data for ANY of the years in the list
+    obs = df.require_variable(variable='Primary Energy', year=[2005, 2010])
+    assert obs is None
+
+    # checking for variables that have data for ALL of the years in the list
+    df = IamDataFrame(test_df.data[1:])
+    exp = pd.DataFrame([['model_a', 'scen_a']], columns=META_IDX)
+
+    obs = df.require_variable(variable='Primary Energy', year=[2005])
+    pdt.assert_frame_equal(obs, exp)
+
+
+def test_validate_pass(test_df):
+    obs = test_df.validate({'Primary Energy': {'up': 10}},
+                           exclude_on_fail=True)
+    assert obs is None
+    assert list(test_df['exclude']) == [False, False]  # none excluded
+
+
+def test_validate_nonexisting(test_df):
+    # checking that a scenario with no relevant value does not fail validation
+    obs = test_df.validate({'Primary Energy|Coal': {'up': 2}},
+                           exclude_on_fail=True)
+    # checking that the return-type is correct
+    pdt.assert_frame_equal(obs, test_df.data[3:4].reset_index(drop=True))
+    # scenario with failed validation excluded, scenario with no value passes
+    assert list(test_df['exclude']) == [True, False]
+
+
+def test_validate_up(test_df):
+    # checking that the return-type is correct
+    obs = test_df.validate({'Primary Energy': {'up': 6.5}})
+    pdt.assert_frame_equal(obs, test_df.data[5:6].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = test_df.validate({'Primary Energy': {'up': 6.5}},
+                           exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, test_df.data[5:6].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, True]
+
+
+def test_validate_lo(test_df):
+    # checking that the return-type is correct
+    obs = test_df.validate({'Primary Energy': {'up': 8, 'lo': 2}})
+    pdt.assert_frame_equal(obs, test_df.data[0:1].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = test_df.validate({'Primary Energy': {'up': 8, 'lo': 2}},
+                           exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, test_df.data[0:1].reset_index(drop=True))
+    assert list(test_df['exclude']) == [True, False]
+
+
+def test_validate_both(test_df):
+    # checking that the return-type is correct
+    obs = test_df.validate({'Primary Energy': {'up': 6.5, 'lo': 2}})
+    pdt.assert_frame_equal(obs, test_df.data[0:6:5].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = test_df.validate({'Primary Energy': {'up': 6.5, 'lo': 2}},
+                           exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, test_df.data[0:6:5].reset_index(drop=True))
+    assert list(test_df['exclude']) == [True, True]
+
+
+def test_validate_year(test_df):
+    # checking that the year filter works as expected
+    obs = test_df.validate({'Primary Energy': {'up': 6, 'year': 2005}})
+    assert obs is None
+
+    # checking that the return-type is correct
+    obs = test_df.validate({'Primary Energy': {'up': 6, 'year': 2010}})
+    pdt.assert_frame_equal(obs, test_df.data[5:6].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, False]
+
+    # checking exclude on fail
+    obs = test_df.validate({'Primary Energy': {'up': 6, 'year': 2010}},
+                           exclude_on_fail=True)
+    pdt.assert_frame_equal(obs, test_df.data[5:6].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, True]
+
+
+def test_validate_top_level(test_df):
+    obs = validate(test_df, criteria={'Primary Energy': {'up': 6}},
+                   exclude_on_fail=True, variable='Primary Energy')
+    pdt.assert_frame_equal(obs, test_df.data[5:6].reset_index(drop=True))
+    assert list(test_df['exclude']) == [False, True]
+
+
+def test_category_none(test_df):
+    test_df.categorize('category', 'Testing', {'Primary Energy': {'up': 0.8}})
+    assert 'category' not in test_df.meta.columns
+
+
+def test_category_pass(test_df):
+    dct = {'model': ['model_a', 'model_a'],
+           'scenario': ['scen_a', 'scen_b'],
+           'category': ['foo', None]}
+    exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
+
+    test_df.categorize('category', 'foo', {'Primary Energy':
+                                           {'up': 6, 'year': 2010}})
+    obs = test_df['category']
+    pd.testing.assert_series_equal(obs, exp)
+
+
+def test_category_top_level(test_df):
+    dct = {'model': ['model_a', 'model_a'],
+           'scenario': ['scen_a', 'scen_b'],
+           'category': ['foo', None]}
+    exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
+
+    categorize(test_df, 'category', 'foo',
+               criteria={'Primary Energy': {'up': 6, 'year': 2010}},
+               variable='Primary Energy')
+    obs = test_df['category']
+    pd.testing.assert_series_equal(obs, exp)