Repository: IAMconsortium/pyam

Issue:
pyam.concat / pyam.append overwrites and not updates year parameter 
the concat command has issues:

```
IAMC_IDX = ['model', 'scenario', 'region', 'variable', 'unit']

TEST_YEARS_PROJ = [2005, 2010]
TEST_YEARS_HIST = [2000]

proj = pd.DataFrame([
    ['model_a', 'scen_a', 'reg', 'PE', 'EJ/yr', 1, 6.],
],
    columns=IAMC_IDX + TEST_YEARS_PROJ,
)
proj = pyam.IamDataFrame(proj)

hist = pd.DataFrame([
    ['source_b', 'scen_b', 'reg', 'PE', 'EJ/yr', 0.5],
],
    columns=IAMC_IDX + TEST_YEARS_HIST,
)
hist = pyam.IamDataFrame(hist)

concat = pyam.concat([hist, proj])

print(concat)
print(concat.year)
```

gives me

```
 <class 'pyam.core.IamDataFrame'>
Index dimensions:
 * model    : model_a, source_b (2)
 * scenario : scen_a, scen_b (2)
Timeseries data coordinates:
   region   : reg (1)
   variable : PE (1)
   unit     : EJ/yr (1)
   year     : 2000, 2005, 2010 (3)
Meta indicators:
   exclude (bool) False (1)

 [2000]
```

(the latter should be [2000, 2005 and 2010]). The issue only arises for .year

Discussion:
danielhuppmann: @ludee @christian-rli @tburandt @suvayu - I'd appreciate your comments!

danielhuppmann: > I left couple of minor suggestions, hope they are helpful.

Thank you @suvayu!
christian-rli: Frankly I am no expert on the OEP data model you are referring to @danielhuppmann . If you're talking about the data model developed in open_MODEX then @jh-RLI might be good at giving some technical insights. Other than that, the PR looks good to my non-expert eyes :)

Patch:
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 5157f8d..409a335 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -34,6 +34,9 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Install Pandoc
+      uses: r-lib/actions/setup-pandoc@v1
+
     - name: Install dependencies and package
       run: |
         python -m pip install --upgrade pip
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index e1ed615..5da496e 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,5 +1,6 @@
 # Next Release
 
+- [#481](https://github.com/IAMconsortium/pyam/pull/481) Enable custom index columns
 - [#477](https://github.com/IAMconsortium/pyam/pull/477) Add a nightly test suite
 - [#471](https://github.com/IAMconsortium/pyam/pull/471) Add a `iiasa.Connection.properties()` function to retrieve scenario audit data
 
diff --git a/pyam/core.py b/pyam/core.py
index d979885..48ecad0 100755
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -45,10 +45,12 @@ from pyam.utils import (
     isstr,
     islistable,
     print_list,
+    DEFAULT_META_INDEX,
     META_IDX,
     IAMC_IDX,
     SORT_IDX,
-    ILLEGAL_COLS
+    ILLEGAL_COLS,
+    _raise_data_error
 )
 from pyam.read_ixmp import read_ix
 from pyam.plotting import PlotAccessor, mpl_args_to_meta_cols
@@ -62,7 +64,7 @@ logger = logging.getLogger(__name__)
 
 
 class IamDataFrame(object):
-    """Scenario timeseries data following the IAMC data format
+    """Scenario timeseries data and meta indicators
 
     The class provides a number of diagnostic features (including validation of
     data, completeness of variables provided), processing tools (e.g.,
@@ -78,6 +80,8 @@ class IamDataFrame(object):
     meta : :class:`pandas.DataFrame`, optional
         A dataframe with suitable 'meta' indicators for the new instance.
         The index will be downselected to scenarios present in `data`.
+    index : list, optional
+        Columns to use for resulting IamDataFrame index.
     kwargs
         If `value=<col>`, melt column `<col>` to 'value' and use `<col>` name
         as 'variable'; or mapping of required columns (:code:`IAMC_IDX`) to
@@ -107,25 +111,33 @@ class IamDataFrame(object):
     This is intended behaviour and consistent with pandas but may be confusing
     for those who are not used to the pandas/Python universe.
     """
-    def __init__(self, data, meta=None, **kwargs):
+    def __init__(self, data, meta=None, index=DEFAULT_META_INDEX, **kwargs):
         """Initialize an instance of an IamDataFrame"""
         if isinstance(data, IamDataFrame):
             if kwargs:
-                msg = 'Invalid arguments `{}` for initializing an IamDataFrame'
-                raise ValueError(msg.format(kwargs))
+                msg = 'Invalid arguments {} for initializing from IamDataFrame'
+                raise ValueError(msg.format(list(kwargs)))
+            if index != data.index.names:
+                msg = f'Incompatible `index={index}` with {type(data)} '
+                raise ValueError(msg + f'(index={data.index.names})')
             for attr, value in data.__dict__.items():
                 setattr(self, attr, value)
         else:
-            self._init(data, meta, **kwargs)
+            self._init(data, meta, index=index, **kwargs)
 
-    def _init(self, data, meta=None, **kwargs):
+    def _init(self, data, meta=None, index=DEFAULT_META_INDEX, **kwargs):
         """Process data and set attributes for new instance"""
         # pop kwarg for meta_sheet_name (prior to reading data from file)
         meta_sheet = kwargs.pop('meta_sheet_name', 'meta')
 
+        # if meta is given explicitly, verify that index matches
+        if meta is not None and not meta.index.names == index:
+            raise ValueError(f'Incompatible `index={index}` with `meta` '
+                             f'(index={meta.index.names})!')
+
         # cast data from pandas
         if isinstance(data, pd.DataFrame) or isinstance(data, pd.Series):
-            _data = format_data(data.copy(), **kwargs)
+            _data = format_data(data.copy(), index=index, **kwargs)
         # read data from ixmp Platform instance
         elif has_ix and isinstance(data, ixmp.TimeSeries):
             # TODO read meta indicators from ixmp
@@ -141,26 +153,24 @@ class IamDataFrame(object):
                 data = Path(data)  # casting str or LocalPath to Path
                 if data.is_file():
                     logger.info(f'Reading file {data}')
-                    _data = read_file(data, **kwargs)
+                    _data = read_file(data, index=index, **kwargs)
                 else:
                     raise FileNotFoundError(f'File {data} does not exist')
             except TypeError:  # `data` cannot be cast to Path
                 msg = 'IamDataFrame constructor not properly called!'
                 raise ValueError(msg)
 
-        _df, self.time_col, self.extra_cols = _data
-        self._LONG_IDX = IAMC_IDX + [self.time_col] + self.extra_cols
-        # cast time_col to desired format
-        self._data = _df.set_index(self._LONG_IDX).value
+        self._data, index, self.time_col, self.extra_cols = _data
+        self._LONG_IDX = self._data.index.names
 
         # define `meta` dataframe for categorization & quantitative indicators
-        self.meta = pd.DataFrame(index=_make_index(self._data))
+        self.meta = pd.DataFrame(index=_make_index(self._data, cols=index))
         self.reset_exclude()
 
-        # merge meta dataframe (if given in kwargs)
+        # if given explicitly, merge meta dataframe after downselecting
         if meta is not None:
-            self.meta = merge_meta(meta.loc[_make_index(self.data)],
-                                   self.meta, ignore_meta_conflict=True)
+            meta = meta.loc[self.meta.index.intersection(meta.index)]
+            self.meta = merge_meta(meta, self.meta, ignore_meta_conflict=True)
 
         # if initializing from xlsx, try to load `meta` table from file
         if meta_sheet and isinstance(data, Path) and data.suffix == '.xlsx':
@@ -168,14 +178,7 @@ class IamDataFrame(object):
             if meta_sheet in excel_file.sheet_names:
                 self.load_meta(excel_file, sheet_name=meta_sheet)
 
-        # add time domain and extra-cols as attributes
-        if self.time_col == 'year':
-            setattr(self, 'year', get_index_levels(self._data, 'year'))
-        else:
-            setattr(self, 'time', pd.Index(
-                get_index_levels(self._data, 'time')))
-        for c in self.extra_cols:
-            setattr(self, c, get_index_levels(self._data, c))
+        self._set_attributes()
 
         # execute user-defined code
         if 'exec' in run_control():
@@ -184,6 +187,25 @@ class IamDataFrame(object):
         # add the `plot` handler
         self.plot = PlotAccessor(self)
 
+    def _set_attributes(self):
+        """Utility function to set attributes"""
+
+        # add time domain as attributes
+        if self.time_col == 'year':
+            setattr(self, 'year', get_index_levels(self._data, 'year'))
+        else:
+            setattr(self, 'time', pd.Index(
+                get_index_levels(self._data, 'time')))
+
+        # set non-standard index columns as attributes
+        for c in self.meta.index.names:
+            if c not in META_IDX:
+                setattr(self, c, get_index_levels(self.meta, c))
+
+        # set extra data columns as attributes
+        for c in self.extra_cols:
+            setattr(self, c, get_index_levels(self._data, c))
+
     def __getitem__(self, key):
         _key_check = [key] if isstr(key) else key
         if set(_key_check).issubset(self.meta.columns):
@@ -220,17 +242,17 @@ class IamDataFrame(object):
         """
         # concatenate list of index dimensions and levels
         info = f'{type(self)}\nIndex dimensions:\n'
-        c1 = max([len(i) for i in self._LONG_IDX]) + 1
+        c1 = max([len(i) for i in self._data.index.names]) + 1
         c2 = n - c1 - 5
         info += '\n'.join(
             [f' * {i:{c1}}: {print_list(get_index_levels(self._data, i), c2)}'
-             for i in META_IDX])
+             for i in self.index.names])
 
         # concatenate list of index of _data (not in META_IDX)
         info += '\nTimeseries data coordinates:\n'
         info += '\n'.join(
             [f'   {i:{c1}}: {print_list(get_index_levels(self._data, i), c2)}'
-             for i in self._LONG_IDX if i not in META_IDX])
+             for i in self._data.index.names if i not in self.index.names])
 
         # concatenate list of (head of) meta indicators and levels/values
         def print_meta_row(m, t, lst):
@@ -285,12 +307,19 @@ class IamDataFrame(object):
     @property
     def model(self):
         """Return the list of (unique) model names"""
-        return get_index_levels(self.meta, 'model')
+        return self._get_meta_index_levels('model')
 
     @property
     def scenario(self):
         """Return the list of (unique) scenario names"""
-        return get_index_levels(self.meta, 'scenario')
+        return self._get_meta_index_levels('scenario')
+
+    def _get_meta_index_levels(self, name):
+        """Return the list of a level from meta"""
+        if name in self.meta.index.names:
+            return get_index_levels(self.meta, name)
+        # in case of non-standard meta.index.names
+        raise KeyError(f'Index `{name}` does not exist!')
 
     @property
     def region(self):
@@ -317,6 +346,9 @@ class IamDataFrame(object):
     @data.setter
     def data(self, df):
         """Set the timeseries data from a long :class:`pandas.DataFrame`"""
+        logger.warning('Setting `data` via the setter can cause'
+                       'inconsistencies with `meta` and other attributes.')
+        deprecation_warning('Please use `IamDataFrame(<data>)` instead!')
         self._data = format_time_col(df, self.time_col)\
             .set_index(self._LONG_IDX).value
 
@@ -456,6 +488,7 @@ class IamDataFrame(object):
                            if i not in ret.extra_cols]
         ret._LONG_IDX = IAMC_IDX + [ret.time_col] + ret.extra_cols
         ret._data = _data.sort_index()
+        ret._set_attributes()
 
         if not inplace:
             return ret
@@ -566,6 +599,7 @@ class IamDataFrame(object):
         df = df.stack()  # long-data to pd.Series
         df.name = 'value'
         ret._data = df.sort_index()
+        ret._set_attributes()
 
         if not inplace:
             return ret
@@ -583,21 +617,27 @@ class IamDataFrame(object):
         ValueError
             "time" is not a column of `self.data`
         """
-        if "time" not in self.data:
-            raise ValueError("time column must be datetime to use this method")
+        if not self.time_col == 'time':
+            raise ValueError('Time domain must be datetime to use this method')
 
         ret = self.copy() if not inplace else self
 
         _data = ret.data
         _data["year"] = _data["time"].apply(lambda x: x.year)
         _data = _data.drop("time", axis="columns")
-        ret._LONG_IDX = [v if v != "time" else "year" for v in ret._LONG_IDX]
+        _index = [v if v != "time" else "year" for v in ret._LONG_IDX]
+
+        rows = _data[_index].duplicated()
+        if any(rows):
+            error_msg = 'Swapping time for year causes duplicates in `data`'
+            _raise_data_error(error_msg, _data[_index])
 
-        if any(_data[ret._LONG_IDX].duplicated()):
-            error_msg = ('swapping time for year will result in duplicate '
-                         'rows in `data`!')
-            raise ValueError(error_msg)
-        ret._data = _data.set_index(IAMC_IDX)
+        # assign data and other attributes
+        ret._LONG_IDX = _index
+        ret._data = _data.set_index(ret._LONG_IDX)
+        ret.time_col = 'year'
+        ret._set_attributes()
+        delattr(ret, 'time')
 
         if not inplace:
             return ret
@@ -1468,6 +1508,7 @@ class IamDataFrame(object):
         if len(idx) == 0:
             logger.warning('Filtered IamDataFrame is empty!')
         ret.meta = ret.meta.loc[idx]
+        ret._set_attributes()
         if not inplace:
             return ret
 
diff --git a/pyam/utils.py b/pyam/utils.py
index 108dc3b..6972d1a 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -20,12 +20,16 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 # common indices
+DEFAULT_META_INDEX = ['model', 'scenario']
 META_IDX = ['model', 'scenario']
 YEAR_IDX = ['model', 'scenario', 'region', 'year']
 IAMC_IDX = ['model', 'scenario', 'region', 'variable', 'unit']
 SORT_IDX = ['model', 'scenario', 'variable', 'year', 'region']
 LONG_IDX = IAMC_IDX + ['year']
 
+# required columns
+REQUIRED_COLS = ['region', 'variable', 'unit']
+
 # illegal terms for data/meta column names to prevent attribute conflicts
 ILLEGAL_COLS = ['data', 'meta']
 
@@ -126,14 +130,14 @@ def read_pandas(path, default_sheet='data', *args, **kwargs):
 
 def read_file(path, *args, **kwargs):
     """Read data from a file"""
-    format_kwargs = {}
     # extract kwargs that are intended for `format_data`
+    format_kwargs = dict(index=kwargs.pop('index'))
     for c in [i for i in IAMC_IDX + ['year', 'time', 'value'] if i in kwargs]:
         format_kwargs[c] = kwargs.pop(c)
     return format_data(read_pandas(path, *args, **kwargs), **format_kwargs)
 
 
-def format_data(df, **kwargs):
+def format_data(df, index, **kwargs):
     """Convert a pandas.Dataframe or pandas.Series to the required format"""
     if isinstance(df, pd.Series):
         df.name = df.name or 'value'
@@ -214,29 +218,33 @@ def format_data(df, **kwargs):
     if conflict_cols:
         msg = f'Column name {conflict_cols} is illegal for timeseries data.\n'
         _args = ', '.join([f"{i}_1='{i}'" for i in conflict_cols])
-        msg += f'Use `IamDataFrame(..., {_args})` to rename at initalization.'
+        msg += f'Use `IamDataFrame(..., {_args})` to rename at initialization.'
         raise ValueError(msg)
 
-    # format columns to lower-case and check that all required columns exist
-    if not set(IAMC_IDX).issubset(set(df.columns)):
-        missing = list(set(IAMC_IDX) - set(df.columns))
-        raise ValueError("missing required columns `{}`!".format(missing))
+    # check that index and required columns exist
+    missing_index = [c for c in index if c not in df.columns]
+    if missing_index:
+        raise ValueError(f'Missing index columns `{missing_index}`!')
+
+    missing_required_col = [c for c in REQUIRED_COLS if c not in df.columns]
+    if missing_required_col:
+        raise ValueError(f'Missing required columns `{missing_required_col}`!')
 
     # check whether data in wide format (IAMC) or long format (`value` column)
     if 'value' in df.columns:
         # check if time column is given as `year` (int) or `time` (datetime)
-        cols = df.columns
-        if 'year' in cols:
+        if 'year' in df.columns:
             time_col = 'year'
-        elif 'time' in cols:
+        elif 'time' in df.columns:
             time_col = 'time'
         else:
-            msg = 'invalid time format, must have either `year` or `time`!'
+            msg = 'Invalid time format, must have either `year` or `time`!'
             raise ValueError(msg)
-        extra_cols = list(set(cols) - set(IAMC_IDX + [time_col, 'value']))
+        extra_cols = [c for c in df.columns
+                      if c not in index + REQUIRED_COLS + [time_col, 'value']]
     else:
         # if in wide format, check if columns are years (int) or datetime
-        cols = set(df.columns) - set(IAMC_IDX)
+        cols = [c for c in df.columns if c not in index + REQUIRED_COLS]
         year_cols, time_cols, extra_cols = [], [], []
         for i in cols:
             try:
@@ -257,7 +265,8 @@ def format_data(df, **kwargs):
         else:
             msg = 'invalid column format, must be either years or `datetime`!'
             raise ValueError(msg)
-        df = pd.melt(df, id_vars=IAMC_IDX + extra_cols, var_name=time_col,
+        cols = index + REQUIRED_COLS + extra_cols
+        df = pd.melt(df, id_vars=cols, var_name=time_col,
                      value_vars=sorted(melt_cols), value_name='value')
 
     # cast value column to numeric and drop nan
@@ -273,7 +282,7 @@ def format_data(df, **kwargs):
         _raise_data_error('empty cells in `data`', df.loc[null_rows])
 
     # check for duplicates and empty data
-    idx_cols = IAMC_IDX + [time_col] + extra_cols
+    idx_cols = index + REQUIRED_COLS + [time_col] + extra_cols
     rows = df[idx_cols].duplicated()
     if any(rows):
         _raise_data_error('duplicate rows in `data`', df.loc[rows, idx_cols])
@@ -282,7 +291,7 @@ def format_data(df, **kwargs):
         logger.warning('Formatted data is empty!')
 
     df = format_time_col(sort_data(df, idx_cols), time_col)
-    return df, time_col, extra_cols
+    return df.set_index(idx_cols).value, index, time_col, extra_cols
 
 
 def format_time_col(data, time_col):
diff --git a/setup.py b/setup.py
index ffcc4e3..81cbd6d 100755
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,7 @@ EXTRA_REQUIREMENTS = {
     'docs': ['sphinx', 'nbsphinx', 'sphinx-gallery', 'cloud_sptheme', 'pillow',
              'sphinxcontrib-bibtex<2.0', 'sphinxcontrib-programoutput',
              'numpydoc', 'openpyxl', 'kaleido']  # docs requires 'tutorials'
+    # GitHub Actions requires pandoc explicitly to build the docs
 }
 
 # building the docs on readthedocs fails with a FileNotFoundError


Test patch:
diff --git a/tests/test_core.py b/tests/test_core.py
index 53e47df..a0b1d72 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -28,6 +28,14 @@ df_filter_by_meta_nonmatching_idx = pd.DataFrame([
 ], columns=['model', 'scenario', 'region', 2010, 2020]
 ).set_index(['model', 'region'])
 
+META_DF = pd.DataFrame([
+    ['model_a', 'scen_a', 1, False],
+    ['model_a', 'scen_b', np.nan, False],
+    ['model_a', 'scen_c', 2, False],
+], columns=META_IDX + ['foo', 'exclude']
+).set_index(META_IDX)
+
+
 df_empty = pd.DataFrame([], columns=IAMC_IDX + [2005, 2010])
 
 
@@ -52,10 +60,9 @@ def test_init_from_iamdf(test_df_year):
 
 def test_init_from_iamdf_raises(test_df_year):
     # casting an IamDataFrame instance again with extra args fails
-    args = dict(model='foo')
-    match = f'Invalid arguments `{args}` for initializing an IamDataFrame'
+    match = "Invalid arguments \['model'\] for initializing from IamDataFrame"
     with pytest.raises(ValueError, match=match):
-        IamDataFrame(test_df_year, **args)
+        IamDataFrame(test_df_year, model='foo')
 
 
 def test_init_df_with_float_cols_raises(test_pd_df):
@@ -97,15 +104,49 @@ def test_init_df_with_extra_col(test_pd_df):
 
     df = IamDataFrame(tdf)
 
-    # check that extra-cols attribute is set correctly
-    assert df.extra_cols == [extra_col]
-
     # check that timeseries data is as expected
     obs = df.timeseries().reset_index()
     exp = tdf[obs.columns]  # get the columns into the right order
     pd.testing.assert_frame_equal(obs, exp)
 
 
+def test_init_df_with_meta(test_pd_df):
+    # pass explicit meta dataframe with a scenario that doesn't exist in data
+    df = IamDataFrame(test_pd_df, meta=META_DF.iloc[[0, 2]][['foo']])
+
+    # check that scenario not existing in data is removed during initialization
+    pd.testing.assert_frame_equal(df.meta, META_DF.iloc[[0, 1]])
+
+
+def test_init_df_with_meta_incompatible_index(test_pd_df):
+    # define a meta dataframe with a non-standard index
+    index = ['source', 'scenario']
+    meta = pd.DataFrame([False, False, False], columns=['exclude'],
+                        index=META_DF.index.rename(index))
+
+    # assert that using an incompatible index for the meta arg raises
+    match = "Incompatible `index=\['model', 'scenario'\]` with `meta` *."
+    with pytest.raises(ValueError, match=match):
+        IamDataFrame(test_pd_df, meta=meta)
+
+
+def test_init_df_with_custom_index(test_pd_df):
+    # rename 'model' column and add a version column to the dataframe
+    test_pd_df.rename(columns={'model': 'source'}, inplace=True)
+    test_pd_df['version'] = [1, 2, 3]
+
+    # initialize with custom index columns, check that index is set correctly
+    index = ['source', 'scenario', 'version']
+    df = IamDataFrame(test_pd_df, index=index)
+    assert df.index.names == index
+
+    # check that index attributes were set correctly and that df.model fails
+    assert df.source == ['model_a']
+    assert df.version == [1, 2, 3]
+    with pytest.raises(KeyError, match='Index `model` does not exist!'):
+        df.model
+
+
 def test_init_empty_message(caplog):
     IamDataFrame(data=df_empty)
     drop_message = (
@@ -245,7 +286,7 @@ def test_index(test_df_year):
 
 
 def test_index_attributes(test_df):
-    # assert that the
+    # assert that the index and data column attributes are set correcty
     assert test_df.model == ['model_a']
     assert test_df.scenario == ['scen_a', 'scen_b']
     assert test_df.region == ['World']
@@ -965,10 +1006,12 @@ def test_swap_time_to_year(test_df, inplace):
 
     if inplace:
         assert obs is None
-        assert compare(test_df, exp).empty
-    else:
-        assert compare(obs, exp).empty
-        assert "year" not in test_df.data.columns
+        obs = test_df
+
+    assert compare(obs, exp).empty
+    assert obs.year == [2005, 2010]
+    with pytest.raises(AttributeError):
+        obs.time
 
 
 @pytest.mark.parametrize("inplace", [True, False])
diff --git a/tests/test_feature_append_rename.py b/tests/test_feature_append_rename.py
index 61c459e..ea92271 100644
--- a/tests/test_feature_append_rename.py
+++ b/tests/test_feature_append_rename.py
@@ -109,6 +109,8 @@ def test_append_extra_col(test_df, shuffle_cols):
     other_data = base_data[base_data["variable"] == "Primary Energy"].copy()
     other_data["variable"] = "Primary Energy|Gas"
     other_df = IamDataFrame(other_data)
+    # cast from FrozenList to list for test
+    other_df._LONG_IDX = list(other_df._LONG_IDX)
 
     if shuffle_cols:
         c1_idx = other_df._LONG_IDX.index("col_1")