Repository: IAMconsortium/pyam
Pull request #55 (2018-03-26 09:05:40)

allow using index as arg in `set_meta()` (refactored from `metadata()`)
This PR allows using an index as an argument to the `set_meta()` function (previously `metadata()`), to assign a value in a meta column to all scenarios satisfying some filters. Previously, I would do the filter, use the index of the meta DataFrame to generate a pd.Series with the value, and pass that to `set_meta()`. This is now internalised in the function.

So the new use case would be `df.set_meta('some_category_name', 'some_column', df.filter('{some filters}').meta.index)`.

Last, setting column values in meta to `uncategorised` (from the pre-refactoring days, currently not operational as far as I can tell) is fixed - if a new column is added to `meta` and the data type of the value is a string, the default of that column is set to `uncategorized`, and to `np.NaN` otherwise.





danielhuppmann: solves #40 
danielhuppmann: Thanks @gidden for the comments!

Resetting the index of either the `data` or `meta` frames does not make sense in our framework (because some functions require the index of `meta` to be `['model', 'scenario']` and `data` needs to have a given set of columns).

Your proposed alternative `meta_index()` could just as easily be achieved using `meta.index` directly. I guess it's easiest to just remove the `index` property and merge the other features of this PR, and then do the more thorough review of nice-to-have pandas features later.
danielhuppmann: As discussed previously, I removed the `IamDataFrame.index` - for now, let's continue to use `IamDataFrame.meta.index` directly.

Also, I refactored the function `metadata()` to `set_meta()`, closes #59 
danielhuppmann: rebased and implemented most suggestions
should be good to be re-reviewed, @gidden 
gidden: Just a typo, then good to go!
danielhuppmann: thanks @gidden for catching the typo!

re the `is scalar()` function, note that the use case here is different than `is scalar()` - pd.Series (and maybe other array types) need to be cast to lists for the later workflow, while strings and scalars can be appended as is. Anyway, I added both your `isscalar()` function (because we'll probably need it at some point anyway) and a new `islistable()` function.

Patch:
diff --git a/pyam/core.py b/pyam/core.py
index c150108..e3f7b8c 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -27,6 +27,7 @@ from pyam.utils import (
     pattern_match,
     years_match,
     isstr,
+    islistable,
     META_IDX,
     IAMC_IDX,
     SORT_IDX,
@@ -248,11 +249,10 @@ class IamDataFrame(object):
         )
 
     def reset_exclude(self):
-        """Reset exclusion assignment for all scenarios to `exclude: False`
-        """
+        """Reset exclusion assignment for all scenarios to `exclude: False`"""
         self.meta['exclude'] = False
 
-    def metadata(self, meta, name=None):
+    def set_meta(self, meta, name=None, index=None):
         """Add metadata columns as pd.Series or list
 
         Parameters
@@ -261,35 +261,57 @@ class IamDataFrame(object):
             column to be added to metadata
             (by `['model', 'scenario']` index if possible)
         name: str
-            category column name (if not given by meta pd.Series.name)
+            meta column name (if not given by meta pd.Series.name)
+        index: pyam.IamDataFrame or pd.MultiIndex, optional
+            index to be used for setting meta column
         """
-        if isinstance(meta, pd.Series) and \
-                set(['model', 'scenario']).issubset(meta.index.names):
-            meta.name = name or meta.name
-            # reduce index dimensions to model-scenario only
-            meta = (meta
-                    .reset_index()
-                    .loc[:, ['model', 'scenario', meta.name]]
-                    .set_index(['model', 'scenario'])
-                    )
-            # raise error if index is not unique
-            if meta.index.duplicated().any():
-                raise ValueError("non-unique ['model', 'scenario'] index!")
-            # check if trying to add model-scenario index not existing in self
-            diff = meta.index.difference(self.meta.index)
-            if not diff.empty:
-                error = "adding metadata for non-existing scenarios '{}'!"
-                raise ValueError(error.format(diff))
-            self.meta = meta.combine_first(self.meta)
-            #  quickfix for pandas.combine_first(), issue #7509
-            self.meta['exclude'] = self.meta['exclude'].astype('bool')
+        if not name and not hasattr(meta, 'name'):
+            raise ValueError('Must pass a name or use a pd.Series')
+
+        # use meta.index if index arg is an IamDataFrame
+        if index is not None and isinstance(index, IamDataFrame):
+            index = index.meta.index
+
+        # create pd.Series from index and meta if provided
+        _meta = meta if index is None else pd.Series(data=meta, index=index,
+                                                     name=name)
+
+        try:
+            append_by_idx = set(META_IDX).issubset(_meta.index.names)
+        except AttributeError:
+            append_by_idx = False
+
+        # if not possible to append by index
+        if not append_by_idx:
+            if islistable(meta):
+                self.meta[name] = list(meta)
+            else:
+                self.meta[name] = meta
             return  # EXIT FUNCTION
 
-        if isinstance(meta, pd.Series):
-            name = name or meta.name
-            meta = meta.tolist()
+        # else append by index
+        name = name or _meta.name
 
-        self.meta[name] = meta
+        # reduce index dimensions to model-scenario only
+        _meta = (
+                    _meta
+                    .reset_index()
+                    .reindex(columns=META_IDX + [name])
+                    .set_index(META_IDX)
+                 )
+
+        # raise error if index is not unique
+        if _meta.index.duplicated().any():
+            raise ValueError("non-unique ['model', 'scenario'] index!")
+
+        # check if trying to add model-scenario index not existing in self
+        diff = _meta.index.difference(self.meta.index)
+        if not diff.empty:
+            error = "adding metadata for non-existing scenarios '{}'!"
+            raise ValueError(error.format(diff))
+
+        self._new_meta_column(name, meta)
+        self.meta[name] = _meta[name].combine_first(self.meta[name])
 
     def categorize(self, name, value, criteria,
                    color=None, marker=None, linestyle=None):
@@ -318,28 +340,28 @@ class IamDataFrame(object):
             if arg:
                 run_control().update({kind: {name: {value: arg}}})
 
-        if criteria == 'uncategorized':
-            self.meta[name].fillna(value, inplace=True)
-            msg = "{} of {} scenarios are uncategorized."
-            logger().info(msg.format(np.sum(self.meta[name] == value),
-                                     len(self.meta)))
-            return  # EXIT FUNCTION
-
         # find all data that matches categorization
         rows = _apply_criteria(self.data, criteria,
                                in_range=True, return_test='all')
         idx = _meta_idx(rows)
 
-        # update metadata dataframe
-        if name not in self.meta:
-            self.meta[name] = np.nan
         if len(idx) == 0:
             logger().info("No scenarios satisfy the criteria")
-        else:
-            self.meta.loc[idx, name] = value
-            msg = '{} scenario{} categorized as `{}: {}`'
-            logger().info(msg.format(len(idx), '' if len(idx) == 1 else 's',
-                                     name, value))
+            return  # EXIT FUNCTION
+
+        # update metadata dataframe
+        self._new_meta_column(name, value)
+        self.meta.loc[idx, name] = value
+        msg = '{} scenario{} categorized as `{}: {}`'
+        logger().info(msg.format(len(idx), '' if len(idx) == 1 else 's',
+                                 name, value))
+
+    def _new_meta_column(self, name, value):
+        """Add a metadata column, set to `uncategorized` if str else np.nan"""
+        if name is None:
+            raise ValueError('cannot add a meta column {}'.format(name))
+        if name not in self.meta:
+            self.meta[name] = 'uncategorized' if isstr(value) else np.nan
 
     def require_variable(self, variable, unit=None, year=None,
                          exclude_on_fail=False):
@@ -414,14 +436,16 @@ class IamDataFrame(object):
         Parameters
         ----------
         mapping: dict
-            for each column where entries should be renamed, provide current name and target name
-            {<column name>: {<current_name_1>: <target_name_1>, <current_name_2>: <target_name_2>}}
+            for each column where entries should be renamed, provide current
+            name and target name
+            {<column name>: {<current_name_1>: <target_name_1>,
+                             <current_name_2>: <target_name_2>}}
         inplace: bool, default False
             if True, do operation inplace and return None
         """
         ret = copy.deepcopy(self) if not inplace else self
         for col in mapping:
-            if not col in ['region', 'variable', 'unit']:
+            if col not in ['region', 'variable', 'unit']:
                 raise ValueError('renaming by {} not supported!'.format(col))
             ret.data.loc[:, col] = self.data.loc[:, col].replace(mapping[col])
 
@@ -435,8 +459,8 @@ class IamDataFrame(object):
         Parameters
         ----------
         conversion_mapping: dict
-            for each unit for which a conversion should be carried out, provide current unit 
-            and target unit and conversion factor
+            for each unit for which a conversion should be carried out,
+            provide current unit and target unit and conversion factor
             {<current unit>: [<target unit>, <conversion factor>]}
         inplace: bool, default False
             if True, do operation inplace and return None
diff --git a/pyam/utils.py b/pyam/utils.py
index bf5b44e..abf7618 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -63,6 +63,16 @@ def isstr(x):
     return isinstance(x, six.string_types)
 
 
+def isscalar(x):
+    """Returns True if x is a scalar"""
+    return not isinstance(x, collections.Iterable) or isstr(x)
+
+
+def islistable(x):
+    """Returns True if x is a list but not a string"""
+    return isinstance(x, collections.Iterable) and not isstr(x)
+
+
 def write_sheet(writer, name, df, index=False):
     """Write a pandas DataFrame to an ExcelWriter,
     auto-formatting column width depending on maxwidth of data and colum header
diff --git a/tutorial/pyam_first_steps-answers.ipynb b/tutorial/pyam_first_steps-answers.ipynb
index 442e9c3..16550a6 100644
--- a/tutorial/pyam_first_steps-answers.ipynb
+++ b/tutorial/pyam_first_steps-answers.ipynb
@@ -1632,7 +1632,7 @@
     "The first cell sets the ``'Temperature'`` categorization to the default `\"uncategorized\"`.\n",
     "This may be helpful in this tutorial if you are going back and forth between cells to reset the assignment.\n",
     "\n",
-    "The function `categorize()` takes `color` and similar arguments, which can then be used by teh plotting library."
+    "The function `categorize()` takes `color` and similar arguments, which can then be used by the plotting library."
    ]
   },
   {
diff --git a/tutorial/pyam_first_steps.ipynb b/tutorial/pyam_first_steps.ipynb
index f952b18..b3eab66 100644
--- a/tutorial/pyam_first_steps.ipynb
+++ b/tutorial/pyam_first_steps.ipynb
@@ -449,9 +449,10 @@
     "We now use the categorization feature of the ``pyam`` package to group scenarios by temperature outcome by the end of the century.\n",
     "\n",
     "The first cell sets the ``'Temperature'`` categorization to the default `\"uncategorized\"`.\n",
-    "This may be helpful in this tutorial if you are going back and forth between cells to reset the assignment.\n",
+    "This is not necessary per se (setting a meta column via the categorization will mark all non-assigned rows as `\"uncategorized\"` (if the value is a string) or `np.nan`.\n",
+    "Still, having this cell may be helpful in this tutorial if you are going back and forth between cells to reset the assignment.\n",
     "\n",
-    "The function `categorize()` takes `color` and similar arguments, which can then be used by teh plotting library."
+    "The function `categorize()` takes `color` and similar arguments, which can then be used by the plotting library."
    ]
   },
   {
@@ -460,7 +461,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df.metadata(meta='uncategorized', name='Temperature')"
+    "df.set_meta(meta='uncategorized', name='Temperature')"
    ]
   },
   {


Test patch:
diff --git a/tests/data/exec.py b/tests/data/exec.py
index cc1b289..6c7b7bb 100644
--- a/tests/data/exec.py
+++ b/tests/data/exec.py
@@ -1,3 +1,3 @@
 
 def do_exec(df):
-    df.metadata('bar', name='foo')
+    df.set_meta('bar', name='foo')
diff --git a/tests/test_core.py b/tests/test_core.py
index e91be53..4ed6993 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -125,8 +125,16 @@ def test_timeseries(test_df):
 
 
 def test_read_pandas():
-    ia = IamDataFrame(os.path.join(TEST_DATA_DIR, 'testing_data_2.csv'))
-    assert list(ia['variable'].unique()) == ['Primary Energy']
+    df = IamDataFrame(os.path.join(TEST_DATA_DIR, 'testing_data_2.csv'))
+    assert list(df.variables()) == ['Primary Energy']
+
+
+def test_filter_meta_index(meta_df):
+    obs = meta_df.filter(scenario='a_scenario2').meta.index
+    exp = pd.MultiIndex(levels=[['a_model'], ['a_scenario2']],
+                        labels=[[0], [0]],
+                        names=['model', 'scenario'])
+    pd.testing.assert_index_equal(obs, exp)
 
 
 def test_meta_idx(meta_df):
@@ -219,21 +227,17 @@ def test_validate_top_level(meta_df):
 
 def test_category_none(meta_df):
     meta_df.categorize('category', 'Testing', {'Primary Energy': {'up': 0.8}})
-    obs = meta_df['category'].values
-    # old usage when default was None
-    # exp = [np.nan, np.nan]
-    # assert list(obs) == exp
-    assert np.isnan(obs).all()
+    assert 'category' not in meta_df.meta.columns
 
 
 def test_category_pass(meta_df):
     dct = {'model': ['a_model', 'a_model'],
            'scenario': ['a_scenario', 'a_scenario2'],
-           'category': ['Testing', np.nan]}
+           'category': ['foo', 'uncategorized']}
     exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
 
-    meta_df.categorize('category', 'Testing', {'Primary Energy':
-                                               {'up': 6, 'year': 2010}})
+    meta_df.categorize('category', 'foo', {'Primary Energy':
+        {'up': 6, 'year': 2010}})
     obs = meta_df['category']
     pd.testing.assert_series_equal(obs, exp)
 
@@ -241,7 +245,7 @@ def test_category_pass(meta_df):
 def test_category_top_level(meta_df):
     dct = {'model': ['a_model', 'a_model'],
            'scenario': ['a_scenario', 'a_scenario2'],
-           'category': ['Testing', np.nan]}
+           'category': ['Testing', 'uncategorized']}
     exp = pd.DataFrame(dct).set_index(['model', 'scenario'])['category']
 
     categorize(meta_df, 'category', 'Testing',
@@ -273,13 +277,13 @@ def test_append(test_df):
     df2 = test_df.append(other=os.path.join(
         TEST_DATA_DIR, 'testing_data_2.csv'))
 
-    obs = test_df['scenario'].unique()
-    exp = ['a_scenario']
-    npt.assert_array_equal(obs, exp)
+    # check that the new meta.index is updated, but not the original one
+    obs = test_df.meta.index.get_level_values(1)
+    npt.assert_array_equal(obs, ['a_scenario'])
 
-    obs = df2['scenario'].unique()
     exp = ['a_scenario', 'append_scenario']
-    npt.assert_array_equal(obs, exp)
+    obs2 = df2.meta.index.get_level_values(1)
+    npt.assert_array_equal(obs2, exp)
 
 
 def test_append_duplicates(test_df):
@@ -297,14 +301,14 @@ def test_interpolate(test_df):
     npt.assert_array_equal(obs, exp)
 
 
-def test_add_metadata_as_named_series(meta_df):
+def test_set_meta_as_named_series(meta_df):
     idx = pd.MultiIndex(levels=[['a_scenario'], ['a_model'], ['a_region']],
                         labels=[[0], [0], [0]],
                         names=['scenario', 'model', 'region'])
 
     s = pd.Series(data=[0.3], index=idx)
     s.name = 'meta_values'
-    meta_df.metadata(s)
+    meta_df.set_meta(s)
 
     idx = pd.MultiIndex(levels=[['a_model'], ['a_scenario', 'a_scenario2']],
                         labels=[[0, 0], [0, 1]], names=['model', 'scenario'])
@@ -315,25 +319,25 @@ def test_add_metadata_as_named_series(meta_df):
     pd.testing.assert_series_equal(obs, exp)
 
 
-def test_add_metadata_non_unique_index_fail(meta_df):
+def test_set_meta_non_unique_index_fail(meta_df):
     idx = pd.MultiIndex(levels=[['a_model'], ['a_scenario'], ['a', 'b']],
                         labels=[[0, 0], [0, 0], [0, 1]],
                         names=['model', 'scenario', 'region'])
     s = pd.Series([0.4, 0.5], idx)
-    pytest.raises(ValueError, meta_df.metadata, s)
+    pytest.raises(ValueError, meta_df.set_meta, s)
 
 
-def test_add_metadata_non_existing_index_fail(meta_df):
+def test_set_meta_non_existing_index_fail(meta_df):
     idx = pd.MultiIndex(levels=[['a_model', 'fail_model'],
                                 ['a_scenario', 'fail_scenario']],
                         labels=[[0, 1], [0, 1]], names=['model', 'scenario'])
     s = pd.Series([0.4, 0.5], idx)
-    pytest.raises(ValueError, meta_df.metadata, s)
+    pytest.raises(ValueError, meta_df.set_meta, s)
 
 
-def test_add_metadata_as_series(meta_df):
+def test_set_meta_as_series(meta_df):
     s = pd.Series([0.3, 0.4])
-    meta_df.metadata(s, 'meta_series')
+    meta_df.set_meta(s, 'meta_series')
 
     idx = pd.MultiIndex(levels=[['a_model'],
                                 ['a_scenario', 'a_scenario2']],
@@ -346,34 +350,56 @@ def test_add_metadata_as_series(meta_df):
     pd.testing.assert_series_equal(obs, exp)
 
 
-def test_add_metadata_as_int(meta_df):
-    meta_df.metadata(3.2, 'meta_int')
+def test_set_meta_as_int(meta_df):
+    meta_df.set_meta(3.2, 'meta_int')
 
     idx = pd.MultiIndex(levels=[['a_model'],
                                 ['a_scenario', 'a_scenario2']],
                         labels=[[0, 0], [0, 1]], names=['model', 'scenario'])
 
-    exp = pd.Series(data=[3.2, 3.2], index=idx)
-    exp.name = 'meta_int'
+    exp = pd.Series(data=[3.2, 3.2], index=idx, name='meta_int')
 
     obs = meta_df['meta_int']
     pd.testing.assert_series_equal(obs, exp)
 
 
-def test_filter_by_metadata_str(meta_df):
-    meta_df.metadata(['testing', 'testing2'], name='category')
+def test_set_meta_as_str(meta_df):
+    meta_df.set_meta('testing', name='meta_str')
+
+    idx = pd.MultiIndex(levels=[['a_model'],
+                                ['a_scenario', 'a_scenario2']],
+                        labels=[[0, 0], [0, 1]], names=['model', 'scenario'])
+
+    exp = pd.Series(data=['testing', 'testing'], index=idx, name='meta_str')
+
+    obs = meta_df['meta_str']
+    pd.testing.assert_series_equal(obs, exp)
+
+
+def test_set_meta_as_str_list(meta_df):
+    meta_df.set_meta(['testing', 'testing2'], name='category')
     obs = meta_df.filter(category='testing')
     assert obs['scenario'].unique() == 'a_scenario'
 
 
-def test_filter_by_metadata_bool(meta_df):
-    meta_df.metadata([True, False], name='exclude')
+def test_set_meta_as_str_by_index(meta_df):
+    idx = pd.MultiIndex(levels=[['a_model'], ['a_scenario']],
+                        labels=[[0], [0]], names=['model', 'scenario'])
+
+    meta_df.set_meta('foo', 'meta_str', idx)
+
+    obs = meta_df['meta_str'].values
+    npt.assert_array_equal(obs, ['foo', 'uncategorized'])
+
+
+def test_filter_by_bool(meta_df):
+    meta_df.set_meta([True, False], name='exclude')
     obs = meta_df.filter(exclude=True)
     assert obs['scenario'].unique() == 'a_scenario'
 
 
-def test_filter_by_metadata_int(meta_df):
-    meta_df.metadata([1, 2], name='value')
+def test_filter_by_int(meta_df):
+    meta_df.set_meta([1, 2], name='value')
     obs = meta_df.filter(value=[1, 3])
     assert obs['scenario'].unique() == 'a_scenario'