Repository: IAMconsortium/pyam
Pull request #43 (2018-03-07 13:40:08)

Exec
allows for sidecar files to be executed, this isn't pretty because we have to alter sys.path. This was the best I could come up with while still supporting Python 2 and 3. See an explanation [here](https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path).


Patch:
diff --git a/pyam/core.py b/pyam/core.py
index 60a0e38..e3cb4a4 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -1,6 +1,8 @@
 import copy
-import os
+import importlib
 import itertools
+import os
+import sys
 
 import numpy as np
 import pandas as pd
@@ -60,6 +62,10 @@ class IamDataFrame(object):
         self.meta = self.data[META_IDX].drop_duplicates().set_index(META_IDX)
         self.reset_exclude()
 
+        # execute user-defined code
+        if 'exec' in run_control():
+            self._execute_run_control()
+
     def __getitem__(self, key):
         _key_check = [key] if isstr(key) else key
         if set(_key_check).issubset(self.meta.columns):
@@ -77,6 +83,21 @@ class IamDataFrame(object):
     def __len__(self):
         return self.data.__len__()
 
+    def _execute_run_control(self):
+        for module_block in run_control()['exec']:
+            fname = module_block['file']
+            functions = module_block['functions']
+
+            dirname = os.path.dirname(fname)
+            if dirname:
+                sys.path.append(dirname)
+
+            module = os.path.basename(fname).split('.')[0]
+            mod = importlib.import_module(module)
+            for func in functions:
+                f = getattr(mod, func)
+                f(self)
+
     def head(self, *args, **kwargs):
         """Identical to pd.DataFrame.head() operating on data"""
         return self.data.head(*args, **kwargs)
@@ -218,8 +239,12 @@ class IamDataFrame(object):
     def timeseries(self):
         """Returns a dataframe in the standard IAMC format
         """
-        return self.data.pivot_table(index=IAMC_IDX,
-                                     columns='year')['value']
+        return (
+            self.data
+            .pivot_table(index=IAMC_IDX, columns='year')
+            .value  # column name
+            .rename_axis(None, axis=1)
+        )
 
     def reset_exclude(self):
         """Reset exclusion assignment for all scenarios to `exclude: False`
@@ -547,6 +572,7 @@ class IamDataFrame(object):
         models = self.meta.index.get_level_values('model')
         fname = fname or run_control()['region_mapping']['default']
         mapping = read_pandas(fname).rename(str.lower, axis='columns')
+        map_col = map_col.lower()
 
         ret = copy.deepcopy(self) if not inplace else self
         _df = ret.data
@@ -556,7 +582,6 @@ class IamDataFrame(object):
         dfs = []
         for model in models:
             df = _df[_df['model'] == model]
-
             _col = region_col or '{}.REGION'.format(model)
             _map = mapping.rename(columns={_col.lower(): 'region'})
             _map = _map[['region', map_col]].dropna().drop_duplicates()


Test patch:
diff --git a/tests/data/exec.py b/tests/data/exec.py
new file mode 100644
index 0000000..cc1b289
--- /dev/null
+++ b/tests/data/exec.py
@@ -0,0 +1,3 @@
+
+def do_exec(df):
+    df.metadata('bar', name='foo')
diff --git a/tests/test_core.py b/tests/test_core.py
index e88f97e..3401910 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -9,7 +9,7 @@ from numpy import testing as npt
 from pyam import IamDataFrame, plotting, validate, categorize, \
     require_variable
 
-from testing_utils import here, meta_df, test_df, reg_df, TEST_DF, TEST_DATA_DIR
+from testing_utils import here, meta_df, test_df, reg_df, TEST_DATA_DIR
 
 
 def test_get_item(test_df):
@@ -24,7 +24,6 @@ def test_model(test_df):
 def test_scenario(test_df):
     exp = pd.Series(data=['a_scenario'], name='scenario')
     pd.testing.assert_series_equal(test_df.scenarios(), exp)
-                                   
 
 
 def test_region(test_df):
@@ -33,7 +32,8 @@ def test_region(test_df):
 
 
 def test_variable(test_df):
-    exp = pd.Series(data=['Primary Energy', 'Primary Energy|Coal'], name='variable')
+    exp = pd.Series(
+        data=['Primary Energy', 'Primary Energy|Coal'], name='variable')
     pd.testing.assert_series_equal(test_df.variables(), exp)
 
 
@@ -153,7 +153,7 @@ def test_validate_up(meta_df):
     assert len(obs) == 1
     assert obs['year'].values[0] == 2010
 
-    assert list(meta_df['exclude']) == [False, False] # assert none excluded
+    assert list(meta_df['exclude']) == [False, False]  # assert none excluded
 
 
 def test_validate_lo(meta_df):
diff --git a/tests/test_run_control.py b/tests/test_run_control.py
new file mode 100644
index 0000000..e386a8c
--- /dev/null
+++ b/tests/test_run_control.py
@@ -0,0 +1,21 @@
+import os
+
+from pyam import IamDataFrame, run_control
+
+from testing_utils import TEST_DATA_DIR, TEST_DF
+
+
+def test_exec():
+    rc = {'exec': [
+        {
+            'file': os.path.join(TEST_DATA_DIR, 'exec.py'),
+            'functions': ['do_exec']
+        },
+    ]}
+
+    run_control().update(rc)
+    df = IamDataFrame(TEST_DF)
+
+    exp = ['bar'] * len(TEST_DF['scenario'].unique())
+    obs = df['foo'].values
+    assert((exp == obs).all())