Repository: sktime/sktime

Issue:
[BUG] ColumnEnsembleForecaster fails with `str` indices
**Describe the bug**
`ColumnEnsembleForecaster` raises an error, when defined over `str` indices (of a dataframe). Specifically [this](https://github.com/sktime/sktime/blob/4194aacaa105ef59807ea56c1bb3497b37eb1613/sktime/forecasting/compose/_column_ensemble.py#L397-L400) error is raised, since the logic there does not account for string indice.

**To Reproduce**
<!--
Add a Minimal, Complete, and Verifiable example (for more details, see e.g. https://stackoverflow.com/help/mcve

If the code is too long, feel free to put it in a public gist and link it in the issue: https://gist.github.com
-->

```python
>>> df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
>>> fc = ColumnEnsembleForecaster([(f"trans_{col}", NaiveForecaster(), col) for col in "ab"])
>>> fc.fit(df)

ValueError: One estimator per column required. Found 2
```

**Expected behavior**
Being able to use `str` indices as describe in the [docstring](https://github.com/sktime/sktime/blob/4194aacaa105ef59807ea56c1bb3497b37eb1613/sktime/forecasting/compose/_column_ensemble.py#L23-L24)
>     forecasters : sktime forecaster, or list of tuples (str, estimator, int or str)
        if tuples, with name = str, estimator is forecaster, index as str or int

**Additional context**
I suspect that the same error will be produced with other `ColumnEnsemble...` objects but I did not check.

**Versions**
<details>
System:
    python: 3.9.5 (default, Nov 23 2021, 15:27:38)  [GCC 9.3.0]
executable: .../bin/python
   machine: Linux-5.4.0-1084-aws-x86_64-with-glibc2.31

Python dependencies:
          pip: 21.2.4
   setuptools: 58.0.4
      sklearn: 0.24.2
       sktime: 0.13.4
  statsmodels: 0.12.2
        numpy: 1.20.3
        scipy: 1.7.1
       pandas: 1.3.4
   matplotlib: 3.4.3
       joblib: 1.0.1
        numba: 0.56.2
     pmdarima: None
      tsfresh: None

</details>

<!-- Thanks for contributing! -->



Discussion:

Resolving patch:
diff --git a/sktime/forecasting/compose/_column_ensemble.py b/sktime/forecasting/compose/_column_ensemble.py
--- a/sktime/forecasting/compose/_column_ensemble.py
+++ b/sktime/forecasting/compose/_column_ensemble.py
@@ -3,12 +3,12 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file).
 """Implements forecaster for applying different univariates by column."""
 
-__author__ = ["GuzalBulatova", "mloning"]
+__author__ = ["GuzalBulatova", "mloning", "fkiraly"]
 __all__ = ["ColumnEnsembleForecaster"]
 
-import numpy as np
 import pandas as pd
 
+from sktime.base._meta import flatten
 from sktime.forecasting.base._base import BaseForecaster
 from sktime.forecasting.base._meta import _HeterogenousEnsembleForecaster
 
@@ -16,12 +16,23 @@
 class ColumnEnsembleForecaster(_HeterogenousEnsembleForecaster):
     """Forecast each series with separate forecaster.
 
-    Applies different univariate forecasters by column.
+    Applies different forecasters by columns.
+
+    `ColumnEnsembleForecaster` is passed forecaster/index pairs, exact syntax below.
+    Index can be single pandas index element, pd.Index, int, str, or list thereof.
+    If iterable (pd.Index, list), refers to multiple columns.
+
+    Behaviour in `fit`, `predict`, `update`:
+    For index pairs f_i, ix_i passed, applies forecaster f_i to column(s) ix_i.
+    `predict` results are concatenated to one container with same columns as in `fit`.
 
     Parameters
     ----------
-    forecasters : sktime forecaster, or list of tuples (str, estimator, int or str)
-        if tuples, with name = str, estimator is forecaster, index as str or int
+    forecasters : sktime forecaster, or list of tuples (str, estimator, int or pd.index)
+        if tuples, with name = str, estimator is forecaster, index as int or index
+        if last element is index, it must be int, str, or pd.Index coercable
+        if last element is int x, and is not in columns, is interpreted as x-th column
+        all columns must be present in an index
 
         If forecaster, clones of forecaster are applied to all columns.
         If list of tuples, forecaster in tuple is applied to column with int/str index
@@ -30,8 +41,11 @@ class ColumnEnsembleForecaster(_HeterogenousEnsembleForecaster):
     --------
     >>> from sktime.forecasting.compose import ColumnEnsembleForecaster
     >>> from sktime.forecasting.exp_smoothing import ExponentialSmoothing
+    >>> from sktime.forecasting.naive import NaiveForecaster
     >>> from sktime.forecasting.trend import PolynomialTrendForecaster
     >>> from sktime.datasets import load_macroeconomic
+
+    Using integers (column iloc refernces= for indexing:
     >>> y = load_macroeconomic()[["realgdp", "realcons"]]
     >>> forecasters = [
     ...     ("trend", PolynomialTrendForecaster(), 0),
@@ -41,6 +55,24 @@ class ColumnEnsembleForecaster(_HeterogenousEnsembleForecaster):
     >>> forecaster.fit(y, fh=[1, 2, 3])
     ColumnEnsembleForecaster(...)
     >>> y_pred = forecaster.predict()
+
+    Using strings for indexing:
+    >>> df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    >>> fc = ColumnEnsembleForecaster(
+    ...     [("foo", NaiveForecaster(), "a"), ("bar", NaiveForecaster(), "b")]
+    ... )
+    >>> fc.fit(df, fh=[1, 42])
+    ColumnEnsembleForecaster(...)
+    >>> y_pred = fc.predict()
+
+    Applying one forecaster to multiple columns, multivariate:
+    >>> df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
+    >>> fc = ColumnEnsembleForecaster(
+    ...    [("ab", NaiveForecaster(), ["a", 1]), ("c", NaiveForecaster(), 2)]
+    ... )
+    >>> fc.fit(df, fh=[1, 42])
+    ColumnEnsembleForecaster(...)
+    >>> y_pred = fc.predict()
     """
 
     _tags = {
@@ -103,7 +135,10 @@ def _forecasters(self, value):
 
     def _coerce_to_pd_index(self, obj):
         """Coerce obj to pandas Index."""
-        if isinstance(obj, int):
+        # replace ints by column names
+        obj = self._get_indices(self._y, obj)
+
+        if isinstance(obj, (int, str)):
             return pd.Index([obj])
         else:
             return pd.Index(obj)
@@ -133,7 +168,8 @@ def _fit(self, y, X=None, fh=None):
             forecaster_ = forecaster.clone()
 
             pd_index = self._coerce_to_pd_index(index)
-            forecaster_.fit(y.iloc[:, pd_index], X, fh)
+
+            forecaster_.fit(y.loc[:, pd_index], X, fh)
             self.forecasters_.append((name, forecaster_, index))
 
         return self
@@ -153,7 +189,7 @@ def _update(self, y, X=None, update_params=True):
         """
         for _, forecaster, index in self.forecasters_:
             pd_index = self._coerce_to_pd_index(index)
-            forecaster.update(y.iloc[:, pd_index], X, update_params=update_params)
+            forecaster.update(y.loc[:, pd_index], X, update_params=update_params)
         return self
 
     def _by_column(self, methodname, **kwargs):
@@ -181,6 +217,8 @@ def _by_column(self, methodname, **kwargs):
             y_preds += [getattr(forecaster, methodname)(**kwargs)]
             keys += [index]
 
+        keys = self._get_indices(self._y, keys)
+
         if col_multiindex:
             y_pred = pd.concat(y_preds, axis=1, keys=keys)
         else:
@@ -360,6 +398,20 @@ def set_params(self, **kwargs):
         self._set_params("_forecasters", **kwargs)
         return self
 
+    def _get_indices(self, y, idx):
+        """Convert integer indices if necessary."""
+
+        def _get_index(y, ix):
+            if isinstance(ix, int) and ix not in y.columns and ix < len(y.columns):
+                return y.columns[ix]
+            else:
+                return ix
+
+        if isinstance(idx, (list, tuple)):
+            return [self._get_indices(y, ix) for ix in idx]
+        else:
+            return _get_index(y, idx)
+
     def _check_forecasters(self, y):
 
         # if a single estimator is passed, replicate across columns
@@ -379,9 +431,13 @@ def _check_forecasters(self, y):
                 " of (string, estimator, int) tuples."
             )
         names, forecasters, indices = zip(*self.forecasters)
-        # defined by MetaEstimatorMixin
+
+        # check names, defined by _HeterogenousEnsembleForecaster
         self._check_names(names)
 
+        # coerce column names to indices in columns
+        indices = self._get_indices(y, indices)
+
         for forecaster in forecasters:
             if not isinstance(forecaster, BaseForecaster):
                 raise ValueError(
@@ -389,15 +445,29 @@ def _check_forecasters(self, y):
                     f"Forecaster."
                 )
 
-        if len(set(indices)) != len(indices):
+        index_flat = flatten(indices)
+        index_set = set(index_flat)
+        not_in_y_idx = index_set.difference(y.columns)
+        y_cols_not_found = set(y.columns).difference(index_set)
+
+        if len(not_in_y_idx) > 0:
             raise ValueError(
-                "One estimator per column required. Found %s unique"
-                " estimators" % len(set(indices))
+                f"Column identifier must be indices in y.columns, or integers within "
+                f"the range of the total number of columns, "
+                f"but found column identifiers that are neither: {list(not_in_y_idx)}"
             )
-        elif not np.array_equal(np.sort(indices), np.arange(len(y.columns))):
+        if len(y_cols_not_found) > 0:
             raise ValueError(
-                "One estimator per column required. Found %s" % len(indices)
+                f"All columns of y must be indexed by column identifiers, but "
+                f"the following columns of y are not indexed: {list(y_cols_not_found)}"
             )
+
+        if len(index_set) != len(index_flat):
+            raise ValueError(
+                f"One estimator per column required. Found {len(index_set)} unique"
+                f" column names in forecasters arg, required {len(index_flat)}"
+            )
+
         return self.forecasters
 
     @classmethod


Test patch:
diff --git a/sktime/forecasting/compose/tests/test_column_ensemble.py b/sktime/forecasting/compose/tests/test_column_ensemble.py
--- a/sktime/forecasting/compose/tests/test_column_ensemble.py
+++ b/sktime/forecasting/compose/tests/test_column_ensemble.py
@@ -3,7 +3,7 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file).
 """Unit tests of ColumnEnsembleForecaster functionality."""
 
-__author__ = ["GuzalBulatova"]
+__author__ = ["GuzalBulatova", "canbooo", "fkiraly"]
 
 import numpy as np
 import pandas as pd
@@ -52,5 +52,25 @@ def test_invalid_forecasters_indices(forecasters):
     """Check if invalid forecaster indices return correct Error."""
     y = pd.DataFrame(np.random.randint(0, 100, size=(100, 3)), columns=list("ABC"))
     forecaster = ColumnEnsembleForecaster(forecasters=forecasters)
-    with pytest.raises(ValueError, match=r"estimator per column"):
+    with pytest.raises(ValueError, match=r"column"):
         forecaster.fit(y, fh=[1, 2])
+
+
+def test_column_ensemble_string_cols():
+    """Check that ColumnEnsembleForecaster works with string columns."""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    fc = ColumnEnsembleForecaster(
+        [(f"trans_{col}", NaiveForecaster(), col) for col in "ab"]
+    )
+    fc.fit(df, fh=[1, 42])
+    fc.predict()
+
+
+def test_column_ensemble_multivariate_and_int():
+    """Check that ColumnEnsembleForecaster works with string columns."""
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
+    fc = ColumnEnsembleForecaster(
+        [("ab", NaiveForecaster(), ["a", 1]), ("c", NaiveForecaster(), 2)]
+    )
+    fc.fit(df, fh=[1, 42])
+    fc.predict()