Repository: IAMconsortium/pyam
Pull request #471 (2020-12-18 15:24:42)

Add an `pyam.iiasa.Connection.properties` function to retrieve audit information
# Please confirm that this PR has done the following:

- [x] Tests Added
- [x] Documentation Added
- ~Name of contributors Added to AUTHORS.rst~
- [x] Description in RELEASE_NOTES.md Added

# Description of PR

This PR adds a `properties` function to retrieve scenario "audit" information (create/update user/timestamp) via the IIASA RestAPI.

Not sure if `properties` is the best name for that function, though, but also no obvious alternative... Maybe `db_meta` or `audit`...

danielhuppmann: pinging @fbenke-pik for a review...
danielhuppmann: Still not sure if `properties()` is the best possible name for that function, but merging to not hold back work...

Patch:
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 493523c..e1ed615 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,6 +1,7 @@
 # Next Release
 
 - [#477](https://github.com/IAMconsortium/pyam/pull/477) Add a nightly test suite
+- [#471](https://github.com/IAMconsortium/pyam/pull/471) Add a `iiasa.Connection.properties()` function to retrieve scenario audit data
 
 # Release v0.10.0
 
@@ -27,6 +28,8 @@ are ordered based on the `run_control()['order']` dictionary by default.
 
 ## Individual updates
 
+- [#464](https://github.com/IAMconsortium/pyam/pull/463) Add `order` arg to `stackplot()`
+- [#466](https://github.com/IAMconsortium/pyam/pull/466) Add a `sankey()`function to create sankey diagrams
 - [#475](https://github.com/IAMconsortium/pyam/pull/475) Add a scatter plot example to the gallery
 - [#473](https://github.com/IAMconsortium/pyam/pull/473) Refactor to plotting API following pandas/matplotlib implementation
 - [#472](https://github.com/IAMconsortium/pyam/pull/472) Add a `sankey()` example to the plotting gallery
diff --git a/pyam/iiasa.py b/pyam/iiasa.py
index 9c3bc96..4e50678 100644
--- a/pyam/iiasa.py
+++ b/pyam/iiasa.py
@@ -235,6 +235,18 @@ class Connection(object):
         _check_response(r, 'Could not retrieve the resource index')
         return pd.read_json(r.content, orient='records')
 
+    @lru_cache()
+    def _query_meta(self, default=True):
+        # TODO: at present this reads in all data for all scenarios,
+        #  it could be sped up in the future to try to query a subset
+        _default = 'true' if default else 'false'
+        add_url = 'runs?getOnlyDefaultRuns={}&includeMetadata=true'
+        url = '/'.join([self._auth_url, add_url.format(_default)])
+        headers = {'Authorization': 'Bearer {}'.format(self._token)}
+        r = requests.get(url, headers=headers)
+        _check_response(r)
+        return pd.read_json(r.content, orient='records')
+
     @property
     @lru_cache()
     def meta_columns(self):
@@ -256,16 +268,7 @@ class Connection(object):
             Any (`model`, `scenario`) without a default version is omitted.
             If :obj:`False`, return all versions.
         """
-        # TODO: at present this reads in all data for all scenarios,
-        #  it could be sped up in the future to try to query a subset
-        _default = 'true' if default else 'false'
-        add_url = 'runs?getOnlyDefaultRuns={}&includeMetadata=true'
-        url = '/'.join([self._auth_url, add_url.format(_default)])
-        headers = {'Authorization': 'Bearer {}'.format(self._token)}
-        r = requests.get(url, headers=headers)
-        _check_response(r)
-        df = pd.read_json(r.content, orient='records')
-
+        df = self._query_meta(default)
         cols = ['version'] if default else ['version', 'is_default']
 
         def extract(row):
@@ -280,6 +283,27 @@ class Connection(object):
         return pd.concat([extract(row) for i, row in df.iterrows()],
                          sort=False)
 
+    def properties(self, default=True):
+        """Return the audit properties of scenarios
+
+        Parameters
+        ----------
+        default : bool, optional
+            Return *only* the default version of each scenario.
+            Any (`model`, `scenario`) without a default version is omitted.
+            If :obj:`False`, return all versions.
+        """
+        _df = self._query_meta(default)
+        audit_cols = ['cre_user', 'cre_date', 'upd_user', 'upd_date']
+        audit_mapping = dict([(i, i.replace('_', 'ate_')) for i in audit_cols])
+        other_cols = ['version'] if default else ['version', 'is_default']
+
+        return (
+            _df[META_IDX + other_cols + audit_cols]
+            .set_index(META_IDX)
+            .rename(columns=audit_mapping)
+        )
+
     def models(self):
         """List all models in the connected resource"""
         return pd.Series(self._query_index()['model'].unique(),


Test patch:
diff --git a/tests/test_iiasa.py b/tests/test_iiasa.py
index cf711aa..7c84ee2 100644
--- a/tests/test_iiasa.py
+++ b/tests/test_iiasa.py
@@ -1,5 +1,4 @@
 import os
-import copy
 import pytest
 import pandas as pd
 import numpy as np
@@ -167,6 +166,24 @@ def test_meta(conn, default):
     pdt.assert_frame_equal(conn.meta(default=default), exp, check_dtype=False)
 
 
+@pytest.mark.parametrize("default", [True, False])
+def test_properties(conn, default):
+    # test that connection returns the correct properties dataframe
+    obs = conn.properties(default=default)
+    if default:
+        exp_cols = ['version']
+        exp = META_DF.loc[META_DF.is_default, exp_cols]
+    else:
+        exp_cols = VERSION_COLS
+        exp = META_DF[exp_cols]
+
+    # assert that the expected audit columns are included
+    for col in ['create_user', 'create_date', 'update_user', 'update_date']:
+        assert col in obs.columns
+    # assert that the values of some columns is as expected
+    pdt.assert_frame_equal(obs[exp_cols], exp, check_dtype=False)
+
+
 @pytest.mark.parametrize("kwargs", [
     {},
     dict(variable='Primary Energy'),