Repository: sktime/sktime
Pull request #5672 (2023-12-28 13:05:22)

[BUG] fix scitype inference utility for all cases
This PR fixes the `scitype` inteference utility for all combinations of `coerce_to_list`, `force_single_scitype`, and adds a generic test for the cases where the tag is a list (multiple/polymorphic), as well as when there is only one tag.

fkiraly: FYI @benHeid, I think the `scitype` utility was not working in polymorphic cases.

Patch:
diff --git a/.all-contributorsrc b/.all-contributorsrc
index 5e234a59f..7f4ece8ba 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -2516,6 +2516,16 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "DManowitz",
+      "name": "David Manowitz",
+      "avatar_url": "https://avatars.githubusercontent.com/u/66927103?v=4",
+      "profile": "https://github.com/DManowitz",
+      "contributions": [
+        "bug",
+        "maintenance"
+      ]
     }
   ]
 }
diff --git a/.github/workflows/validate_datasets.yml b/.github/workflows/validate_datasets.yml
index ee51b8ee1..fb0ca77fc 100644
--- a/.github/workflows/validate_datasets.yml
+++ b/.github/workflows/validate_datasets.yml
@@ -40,6 +40,10 @@ jobs:
       - name: unit test step
         run: python3 -m pytest -m "not datadownload" sktime/datasets
         shell: bash
+      - name: upload coverage step
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},datasets
   test_external_data:
     name: test data downloaded from external sources
     strategy:
@@ -75,3 +79,7 @@ jobs:
       - name: unit test step
         run: python3 -m pytest -m "datadownload" sktime/datasets
         shell: bash
+      - name: upload coverage step
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},datasets
diff --git a/.github/workflows/validate_extras.yml b/.github/workflows/validate_extras.yml
index 69d205af7..4de86aa94 100644
--- a/.github/workflows/validate_extras.yml
+++ b/.github/workflows/validate_extras.yml
@@ -21,6 +21,7 @@ jobs:
           - ubuntu-latest
           - windows-latest
         sktime-extra:
+          - all_extras_pandas2
           - alignment
           - annotation
           - classification
diff --git a/pyproject.toml b/pyproject.toml
index 0011d7662..27371869c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -109,7 +109,7 @@ all_extras = [
   'tbats>=1.1; python_version < "3.12"',
   'tensorflow; python_version < "3.12"',
   'tsfresh>=0.17; python_version < "3.12"',
-  'tslearn<0.6.0,>=0.5.2; python_version < "3.11"',
+  'tslearn<0.7.0,!=0.6.0,>=0.5.2; python_version < "3.11"',
   "xarray",
 ]
 all_extras_pandas2 = [
@@ -144,7 +144,7 @@ all_extras_pandas2 = [
   'tbats>=1.1; python_version < "3.12"',
   'tensorflow; python_version < "3.12"',
   'tsfresh>=0.17; python_version < "3.12"',
-  'tslearn<0.6.0,>=0.5.2; python_version < "3.11"',
+  'tslearn<0.7.0,!=0.6.0,>=0.5.2; python_version < "3.11"',
   "xarray",
 ]
 
@@ -174,7 +174,7 @@ classification = [
 ]
 clustering = [
   'numba<0.59,>=0.53; python_version < "3.12"',
-  'tslearn<0.6.3,>=0.5.2; python_version < "3.12"',
+  'tslearn<0.7.0,!=0.6.0,>=0.5.2; python_version < "3.12"',
 ]
 forecasting = [
   "arch<6.3,>=5.6",
diff --git a/sktime/base/_base_panel.py b/sktime/base/_base_panel.py
index 9cffa2a41..8faa1ba9c 100644
--- a/sktime/base/_base_panel.py
+++ b/sktime/base/_base_panel.py
@@ -265,22 +265,29 @@ class BasePanelMixin(BaseEstimator):
         )
         return X
 
-    def _check_y(self, y=None):
+    def _check_y(self, y=None, return_to_mtype=False):
         """Check and coerce X/y for fit/transform functions.
 
         Parameters
         ----------
         y : pd.DataFrame, pd.Series or np.ndarray
+        return_to_mtype : bool
+            whether to return the mtype of y output
 
         Returns
         -------
-        y : object of sktime compatible time series type
+        y_inner : object of sktime compatible time series type
             can be Series, Panel, Hierarchical
         y_metadata : dict
             metadata of y, retured by check_is_scitype
+        y_mtype : str, only returned if return_to_mtype=True
+            mtype of y_inner, after convert
         """
         if y is None:
-            return None
+            if return_to_mtype:
+                return None, None, None
+            else:
+                return None, None
 
         capa_multioutput = self.get_tag("capability:multioutput")
         y_inner_mtype = self.get_tag("y_inner_mtype")
@@ -314,17 +321,24 @@ class BasePanelMixin(BaseEstimator):
                 store=self._converter_store_y,
             )
             y_vec = VectorizedDF([y_df], iterate_cols=True)
-            return y_vec, y_metadata
+            if return_to_mtype:
+                return y_vec, y_metadata, "pd_DataFrame_Table"
+            else:
+                return y_vec, y_metadata
 
-        y_inner = convert(
+        y_inner, y_inner_mtype = convert(
             y,
             from_type=y_mtype,
             to_type=y_inner_mtype,
             as_scitype="Table",
             store=self._converter_store_y,
+            return_to_mtype=True,
         )
 
-        return y_inner, y_metadata
+        if return_to_mtype:
+            return y_inner, y_metadata, y_inner_mtype
+        else:
+            return y_inner, y_metadata
 
     def _convert_output_y(self, y):
         """Convert output y to original format.
@@ -332,6 +346,7 @@ class BasePanelMixin(BaseEstimator):
         Parameters
         ----------
         y : np.ndarray or pd.DataFrame
+            output to convert
 
         Returns
         -------
@@ -346,9 +361,18 @@ class BasePanelMixin(BaseEstimator):
             output_mtype = "numpy1D"
             converter_store = None
 
+        # inner return mtype is what we convert from
+        # special treatment for 1D numpy array
+        # this can be returned in composites due to
+        # current downwards compatible choice "1D return is always numpy"
+        if isinstance(y, np.ndarray) and y.ndim == 1:
+            inner_return_mtype = "numpy1D"
+        else:
+            inner_return_mtype = self._y_inner_mtype
+
         y = convert(
             y,
-            from_type=self.get_tag("y_inner_mtype"),
+            from_type=inner_return_mtype,
             to_type=output_mtype,
             as_scitype="Table",
             store=converter_store,
diff --git a/sktime/classification/base.py b/sktime/classification/base.py
index 8f218969c..967fccf57 100644
--- a/sktime/classification/base.py
+++ b/sktime/classification/base.py
@@ -180,8 +180,9 @@ class BaseClassifier(BasePanelMixin):
         start = int(round(time.time() * 1000))
 
         # check and convert y for multioutput vectorization
-        y, y_metadata = self._check_y(y)
+        y, y_metadata, y_inner_mtype = self._check_y(y, return_to_mtype=True)
         self._y_metadata = y_metadata
+        self._y_inner_mtype = y_inner_mtype
         self._is_vectorized = isinstance(y, VectorizedDF)
 
         if self._is_vectorized:
diff --git a/sktime/classification/dictionary_based/_boss.py b/sktime/classification/dictionary_based/_boss.py
index 88889a05a..8a1eaae13 100644
--- a/sktime/classification/dictionary_based/_boss.py
+++ b/sktime/classification/dictionary_based/_boss.py
@@ -667,6 +667,7 @@ class IndividualBOSS(BaseClassifier):
         new_boss.classes_ = self.classes_
         new_boss._class_dictionary = self._class_dictionary
         new_boss._y_metadata = self._y_metadata
+        new_boss._y_inner_mtype = self._y_inner_mtype
         new_boss._is_fitted = True
 
         return new_boss
diff --git a/sktime/classification/early_classification/base.py b/sktime/classification/early_classification/base.py
index f0702c5af..84b72c677 100644
--- a/sktime/classification/early_classification/base.py
+++ b/sktime/classification/early_classification/base.py
@@ -630,20 +630,26 @@ class BaseEarlyClassifier(BaseEstimator, ABC):
         _convert_X = BaseClassifier._convert_X
         return _convert_X(self, X, X_mtype)
 
-    def _check_y(self, y=None):
+    def _check_y(self, y=None, return_to_mtype=False):
         """Check and coerce X/y for fit/transform functions.
 
         Parameters
         ----------
         y : pd.DataFrame, pd.Series or np.ndarray
+        return_to_mtype : bool
+            whether to return the mtype of y output
 
         Returns
         -------
-        y : object of sktime compatible time series type
+        y_inner : object of sktime compatible time series type
             can be Series, Panel, Hierarchical
+        y_metadata : dict
+            metadata of y, retured by check_is_scitype
+        y_mtype : str, only returned if return_to_mtype=True
+            mtype of y_inner, after convert
         """
         _check_y = BaseClassifier._check_y
-        return _check_y(self, y)
+        return _check_y(self, y, return_to_mtype=return_to_mtype)
 
     def _check_input(self, X, y=None, enforce_min_instances=1, return_metadata=True):
         """Check whether input X and y are valid formats with minimum data.
diff --git a/sktime/classification/feature_based/_tsfresh_classifier.py b/sktime/classification/feature_based/_tsfresh_classifier.py
index 50216d7bd..d5fcede7b 100644
--- a/sktime/classification/feature_based/_tsfresh_classifier.py
+++ b/sktime/classification/feature_based/_tsfresh_classifier.py
@@ -149,6 +149,7 @@ class TSFreshClassifier(BaseClassifier):
             self._estimator.n_jobs = self._threads_to_use
 
         X_t = self._transformer.fit_transform(X, y)
+        self._Xt_colnames = X_t.columns
 
         if X_t.shape[1] == 0:
             warn(
@@ -182,7 +183,9 @@ class TSFreshClassifier(BaseClassifier):
         if self._return_majority_class:
             return np.full(X.shape[0], self.classes_[self._majority_class])
 
-        return self._estimator.predict(self._transformer.transform(X))
+        X_t = self._transformer.transform(X)
+        X_t = X_t.reindex(self._Xt_colnames, axis=1, fill_value=0)
+        return self._estimator.predict(X_t)
 
     def _predict_proba(self, X) -> np.ndarray:
         """Predict class probabilities for n instances in X.
@@ -207,7 +210,9 @@ class TSFreshClassifier(BaseClassifier):
             return self._estimator.predict_proba(self._transformer.transform(X))
         else:
             dists = np.zeros((X.shape[0], self.n_classes_))
-            preds = self._estimator.predict(self._transformer.transform(X))
+            X_t = self._transformer.transform(X)
+            X_t = X_t.reindex(self._Xt_colnames, axis=1, fill_value=0)
+            preds = self._estimator.predict(X_t)
             for i in range(0, X.shape[0]):
                 dists[i, self._class_dictionary[preds[i]]] = 1
             return dists
diff --git a/sktime/datatypes/_convert.py b/sktime/datatypes/_convert.py
index fcc46c2f7..eac9d2fe8 100644
--- a/sktime/datatypes/_convert.py
+++ b/sktime/datatypes/_convert.py
@@ -301,8 +301,8 @@ def _get_first_mtype_of_same_scitype(from_mtype, to_mtypes, varname="to_mtypes")
     ]
     if len(same_scitype_mtypes) == 0:
         raise TypeError(
-            f"{varname} contains no mtype compatible with the scitype of obj,"
-            f"which is {scitype}"
+            f"{varname} contains no mtype compatible with the scitype of obj, "
+            f"which is {scitype}. Value of {varname} is: {to_mtypes}"
         )
     to_type = same_scitype_mtypes[0]
     return to_type
diff --git a/sktime/forecasting/base/_base.py b/sktime/forecasting/base/_base.py
index f22c39e6b..632462087 100644
--- a/sktime/forecasting/base/_base.py
+++ b/sktime/forecasting/base/_base.py
@@ -368,11 +368,7 @@ class BaseForecaster(BaseEstimator):
 
         return self
 
-    def predict(
-        self,
-        fh=None,
-        X=None,
-    ):
+    def predict(self, fh=None, X=None):
         """Forecast time series at future horizon.
 
         State required:
@@ -421,7 +417,7 @@ class BaseForecaster(BaseEstimator):
         # convert to output mtype, identical with last y mtype seen
         y_out = convert_to(
             y_pred,
-            self._y_mtype_last_seen,
+            self._y_metadata["mtype"],
             store=self._converter_store_y,
             store_behaviour="freeze",
         )
@@ -1085,7 +1081,7 @@ class BaseForecaster(BaseEstimator):
         # convert to output mtype, identical with last y mtype seen
         y_pred = convert_to(
             y_pred,
-            self._y_mtype_last_seen,
+            self._y_metadata["mtype"],
             store=self._converter_store_y,
             store_behaviour="freeze",
         )
@@ -1166,7 +1162,7 @@ class BaseForecaster(BaseEstimator):
         y_pred = self.predict(fh=fh, X=X)
 
         if not type(y_pred) is type(y):
-            y = convert_to(y, self._y_mtype_last_seen)
+            y = convert_to(y, self._y_metadata["mtype"])
 
         y_res = y - y_pred
 
@@ -1313,7 +1309,7 @@ class BaseForecaster(BaseEstimator):
 
         Writes to self
         --------------
-        _y_mtype_last_seen : str, mtype of y
+        _y_metadata : dict with str keys, metadata from checking y
         _converter_store_y : dict, metadata from conversion for back-conversion
         """
         if X is None and y is None:
@@ -1364,7 +1360,7 @@ class BaseForecaster(BaseEstimator):
         # checking y
         if y is not None:
             # request only required metadata from checks
-            y_metadata_required = []
+            y_metadata_required = ["n_features", "feature_names"]
             if self.get_tag("scitype:y") != "both":
                 y_metadata_required += ["is_univariate"]
             if not self.get_tag("handles-missing-data"):
@@ -1392,6 +1388,7 @@ class BaseForecaster(BaseEstimator):
                 raise TypeError(msg + ", ".join(mtypes_messages))
 
             y_scitype = y_metadata["scitype"]
+            self._y_metadata = y_metadata
             self._y_mtype_last_seen = y_metadata["mtype"]
 
             req_vec_because_rows = y_scitype not in y_inner_scitype
@@ -1921,13 +1918,13 @@ class BaseForecaster(BaseEstimator):
             )
             # we need to overwrite the mtype last seen and converter store, since the _y
             #    may have been converted
-            mtype_last_seen = self._y_mtype_last_seen
+            y_metadata = self._y_metadata
             _converter_store_y = self._converter_store_y
             # refit with updated data, not only passed data
             self.fit(y=self._y, X=self._X, fh=self._fh)
             # todo: should probably be self._fit, not self.fit
             # but looping to self.fit for now to avoid interface break
-            self._y_mtype_last_seen = mtype_last_seen
+            self._y_metadata = y_metadata
             self._converter_store_y = _converter_store_y
 
         # if update_params=False, and there are no components, do nothing
@@ -2020,14 +2017,9 @@ class BaseForecaster(BaseEstimator):
         # change the column labels (multiindex) to the format for intervals
         # idx returned by _predict_quantiles is
         #   2-level MultiIndex with variable names, alpha
-        idx = pred_int.columns
-        # variable names (unique, in same order)
-        var_names = idx.get_level_values(0).unique()
-
         # idx returned by _predict_interval should be
         #   3-level MultiIndex with variable names, coverage, lower/upper
-        int_idx = pd.MultiIndex.from_product([var_names, coverage, ["lower", "upper"]])
-
+        int_idx = self._get_columns(method="predict_interval", coverage=coverage)
         pred_int.columns = int_idx
 
         return pred_int
@@ -2093,14 +2085,9 @@ class BaseForecaster(BaseEstimator):
             # change the column labels (multiindex) to the format for intervals
             # idx returned by _predict_interval is
             #   3-level MultiIndex with variable names, coverage, lower/upper
-            idx = pred_int.columns
-            # variable names (unique, in same order)
-            var_names = idx.get_level_values(0).unique()
-
             # idx returned by _predict_quantiles should be
             #   is 2-level MultiIndex with variable names, alpha
-            int_idx = pd.MultiIndex.from_product([var_names, alpha])
-
+            int_idx = self._get_columns(method="predict_quantiles", alpha=alpha)
             pred_int.columns = int_idx
 
         elif implements_proba:
@@ -2170,8 +2157,9 @@ class BaseForecaster(BaseEstimator):
             if fh.is_relative:
                 fh = fh.to_absolute(self.cutoff)
             pred_var.index = fh.to_pandas()
+
             if isinstance(self._y, pd.DataFrame):
-                pred_var.columns = self._y.columns
+                pred_var.columns = self._get_columns(method="predict_var")
 
             return pred_var
 
@@ -2339,29 +2327,74 @@ class BaseForecaster(BaseEstimator):
             for i in range(len(y_preds)):
                 y_preds[i] = convert_to(
                     y_preds[i],
-                    self._y_mtype_last_seen,
+                    self._y_metadata["mtype"],
                     store=self._converter_store_y,
                     store_behaviour="freeze",
                 )
         return _format_moving_cutoff_predictions(y_preds, cutoffs)
 
-    def _get_varnames(self):
+    def _get_varnames(self, y=None):
         """Return variable column for DataFrame-like returns.
 
-        Developer note: currently a helper for predict_interval, predict_quantiles,
-        valid only in the univariate case. Can be extended later.
+        Primarily used as helper for probabilistic predict-like methods.
+        Assumes that _check_X_y has been called, and self._y_metadata set.
+
+        Parameter
+        ---------
+        y : ignored, present for downwards compatibility
+
+        Returns
+        -------
+        varnames : iterable of integer or str variable names
+            can be list or pd.Index
+            variable names for DataFrame-like returns
+            identical to self._y_varnames if this attribute exists
+        """
+        featnames = self._y_metadata["feature_names"]
+        return featnames
+
+    def _get_columns(self, method="predict", **kwargs):
+        """Return column names for DataFrame-like returns.
+
+        Primarily used as helper for probabilistic predict-like methods.
+        Assumes that _check_X_y has been called, and self._y_metadata set.
+
+        Parameter
+        ---------
+        method : str, optional (default="predict")
+            method for which to return column names
+            one of "predict", "predict_interval", "predict_quantiles", "predict_var"
+        kwargs : dict
+            additional keyword arguments passed to private method
+            important: args to private method, e.g., _predict, _predict_interval
+
+        Returns
+        -------
+        columns : pd.Index
+            column names
         """
-        y = self._y
-        if isinstance(y, pd.Series):
-            var_name = self._y.name
-        elif isinstance(y, pd.DataFrame):
-            return y.columns
+        featnames = self._get_varnames()
+
+        if method in ["predict", "predict_var"]:
+            return featnames
         else:
-            var_name = 0
-        if var_name is None:
-            var_name = 0
+            assert method in ["predict_interval", "predict_quantiles"]
+
+        if method == "predict_interval":
+            coverage = kwargs.get("coverage", None)
+            if coverage is None:
+                raise ValueError(
+                    "coverage must be passed to _get_columns for predict_interval"
+                )
+            return pd.MultiIndex.from_product([featnames, coverage, ["lower", "upper"]])
 
-        return [var_name]
+        if method == "predict_quantiles":
+            alpha = kwargs.get("alpha", None)
+            if alpha is None:
+                raise ValueError(
+                    "alpha must be passed to _get_columns for predict_quantiles"
+                )
+            return pd.MultiIndex.from_product([featnames, alpha])
 
 
 def _format_moving_cutoff_predictions(y_preds, cutoffs):
diff --git a/sktime/registry/_scitype.py b/sktime/registry/_scitype.py
index 8ef58e6ac..e00377ffb 100644
--- a/sktime/registry/_scitype.py
+++ b/sktime/registry/_scitype.py
@@ -18,15 +18,16 @@ def scitype(obj, force_single_scitype=True, coerce_to_list=False):
         if True, only the *first* scitype found will be returned
         order is determined by the order in BASE_CLASS_REGISTER
     coerce_to_list : bool, optional, default = False
-        whether return should be coerced to list, even if only one scitype is identified
+        determines the return type: if True, returns a single str,
+        if False, returns a list of str
 
     Returns
     -------
     scitype : str, or list of str of sktime scitype strings from BASE_CLASS_REGISTER
         str, sktime scitype string, if exactly one scitype can be determined for obj
-            or force_single_scitype is True, and if coerce_to_list is False
+        or force_single_scitype is True, and if coerce_to_list is False
         list of str, of scitype strings, if more than one scitype are determined,
-            or if coerce_to_list is True
+        or if coerce_to_list is True
         obj has scitype if it inherits from class in same row of BASE_CLASS_REGISTER
 
     Raises
@@ -40,10 +41,14 @@ def scitype(obj, force_single_scitype=True, coerce_to_list=False):
         else:
             tag_type = obj.get_tag("object_type", None, raise_error=False)
         if tag_type is not None:
-            if coerce_to_list and not isinstance(tag_type, list):
-                return [tag_type]
-            else:
+            if not isinstance(tag_type, list):
+                tag_type = [tag_type]
+            if force_single_scitype and len(tag_type) > 1:
+                tag_type = [tag_type[0]]
+            if coerce_to_list:
                 return tag_type
+            else:
+                return tag_type[0]
 
     # if the tag is not present, determine scitype from legacy base class logic
     if isclass(obj):
diff --git a/sktime/regression/base.py b/sktime/regression/base.py
index f89e1aa86..5323da101 100644
--- a/sktime/regression/base.py
+++ b/sktime/regression/base.py
@@ -174,8 +174,9 @@ class BaseRegressor(BasePanelMixin):
         start = int(round(time.time() * 1000))
 
         # check and convert y for multioutput vectorization
-        y, y_metadata = self._check_y(y)
+        y, y_metadata, y_inner_mtype = self._check_y(y, return_to_mtype=True)
         self._y_metadata = y_metadata
+        self._y_inner_mtype = y_inner_mtype
         self._is_vectorized = isinstance(y, VectorizedDF)
 
         if self._is_vectorized:
diff --git a/sktime/transformations/base.py b/sktime/transformations/base.py
index 23005244c..16c062707 100644
--- a/sktime/transformations/base.py
+++ b/sktime/transformations/base.py
@@ -1303,9 +1303,19 @@ class BaseTransformer(BaseEstimator):
                 n_fit = n * m
                 if n_trafos != n_fit:
                     raise RuntimeError(
+                        f"{type(self).__name__} is a transformer that applies per "
+                        "individual time series, and broadcasts across instances. "
+                        f"In fit, {type(self).__name__} makes one fit per instance, "
+                        "and applies that fit to the instance with the same index in "
+                        "transform. Vanilla use therefore requires the same number "
+                        "of instances in fit and transform, but"
                         "found different number of instances in transform than in fit. "
                         f"number of instances seen in fit: {n_fit}; "
-                        f"number of instances seen in transform: {n_trafos}"
+                        f"number of instances seen in transform: {n_trafos}. "
+                        "For fit/transforming per instance, e.g., for pre-processinng "
+                        "in a time series classification, regression or clustering "
+                        "pipeline, wrap this transformer in "
+                        "FitInTransform, from sktime.transformations.compose."
                     )
 
                 transformers_ = self.transformers_


Test patch:
diff --git a/.github/actions/test-base/action.yml b/.github/actions/test-base/action.yml
index c87d95235..a0359c79e 100644
--- a/.github/actions/test-base/action.yml
+++ b/.github/actions/test-base/action.yml
@@ -29,9 +29,10 @@ runs:
       run: python3 -m pip install .[tests]
       shell: bash
     - name: unit test step
-      run: python3 -m pytest sktime/base --matrixdesign ${{ inputs.sub-sample-estimators }} --only_changed_modules ${{ inputs.test-affected-estimators }}
+      run: >-
+        python3
+        -m pytest
+        sktime/base
+        --matrixdesign ${{ inputs.sub-sample-estimators }}
+        --only_changed_modules ${{ inputs.test-affected-estimators }}
       shell: bash
-    - name: test coverage step
-      uses: codecov/codecov-action@v3
-      with:
-        flags: ${{ inputs.python-version-identifier }},base
diff --git a/.github/actions/test-component/action.yml b/.github/actions/test-component/action.yml
index 4b154aa1f..297544d5a 100644
--- a/.github/actions/test-component/action.yml
+++ b/.github/actions/test-component/action.yml
@@ -32,9 +32,10 @@ runs:
       run: python3 -m pip install .[${{ inputs.sktime-component-identifier }},tests]
       shell: bash
     - name: unit test step
-      run: python3 -m pytest sktime/${{ inputs.sktime-component-identifier }} --matrixdesign ${{ inputs.sub-sample-estimators }} --only_changed_modules ${{ inputs.test-affected-estimators }}
+      run: >-
+        python3
+        -m pytest
+        sktime/${{ inputs.sktime-component-identifier }}
+        --matrixdesign ${{ inputs.sub-sample-estimators }}
+        --only_changed_modules ${{ inputs.test-affected-estimators }}
       shell: bash
-    - name: test coverage step
-      uses: codecov/codecov-action@v3
-      with:
-        flags: ${{ inputs.python-version-identifier }},${{ inputs.sktime-component-identifier }}
diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml
index 334502190..a70e4bf9e 100644
--- a/.github/workflows/test_all.yml
+++ b/.github/workflows/test_all.yml
@@ -1,4 +1,4 @@
-name: weekly test all estimators
+name: test everything
 on:
   schedule:
     - cron: 0 0 * * 0
@@ -23,7 +23,7 @@ jobs:
         shell: bash
   test_base:
     needs: code_quality
-    name: test base module
+    name: test base component
     strategy:
       fail-fast: false
       matrix:
@@ -47,9 +47,13 @@ jobs:
           python-version-identifier: ${{ matrix.python-version }}
           sub-sample-estimators: "False"
           test-affected-estimators: "False"
+      - name: upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},base,complete
   test_components_with_extras:
     needs: code_quality
-    name: test module
+    name: test components with extras
     strategy:
       fail-fast: false
       matrix:
@@ -84,9 +88,13 @@ jobs:
           python-version-identifier: ${{ matrix.python-version }}
           sub-sample-estimators: "False"
           test-affected-estimators: "False"
+      - name: upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},${{ matrix.sktime-component }},complete
   test_components_without_extras:
     needs: code_quality
-    name: test all components without extras
+    name: test components without extras
     strategy:
       fail-fast: false
       matrix:
@@ -115,8 +123,24 @@ jobs:
       - name: install core, test and all soft dependencies
         run: python3 -m pip install .[all_extras_pandas2,tests]
       - name: run unit tests
-        run: python3 -m pytest sktime --ignore sktime/alignment --ignore sktime/annotation --ignore sktime/classification --ignore sktime/clustering --ignore sktime/datasets --ignore sktime/forecasting --ignore sktime/networks --ignore sktime/param_est --ignore sktime/regression --ignore sktime/transformations --matrixdesign False --only_changed_modules False
-      - name: test coverage
+        run: >-
+          python3
+          -m pytest
+          sktime
+          --ignore sktime/base
+          --ignore sktime/datasets
+          --ignore sktime/alignment
+          --ignore sktime/annotation
+          --ignore sktime/classification
+          --ignore sktime/clustering
+          --ignore sktime/forecasting
+          --ignore sktime/networks
+          --ignore sktime/param_est
+          --ignore sktime/regression
+          --ignore sktime/transformations
+          --matrixdesign False
+          --only_changed_modules False
+      - name: upload coverage
         uses: codecov/codecov-action@v3
         with:
-          flags: ${{ matrix.python-version }},components_without_extras
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},components_without_extras,complete
diff --git a/.github/workflows/test_base.yml b/.github/workflows/test_base.yml
index f1d15c9d9..5b77ea9e7 100644
--- a/.github/workflows/test_base.yml
+++ b/.github/workflows/test_base.yml
@@ -1,9 +1,9 @@
-name: test base module
+name: test base component
 on:
   workflow_call:
 jobs:
   detect:
-    name: check for changes in base module
+    name: detect changes in base component
     runs-on: ubuntu-latest
     permissions:
       pull-requests: read
@@ -18,7 +18,7 @@ jobs:
               - sktime/base/**
   test:
     needs: detect
-    name: test base module
+    name: test base component
     if: ${{ needs.detect.outputs.base == 'true' }}
     strategy:
       fail-fast: false
@@ -43,3 +43,7 @@ jobs:
           python-version-identifier: ${{ matrix.python-version }}
           sub-sample-estimators: "True"
           test-affected-estimators: "True"
+      - name: upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},base,incomplete
diff --git a/.github/workflows/test_code_quality.yml b/.github/workflows/test_code_quality.yml
index 48e90042a..4020541f8 100644
--- a/.github/workflows/test_code_quality.yml
+++ b/.github/workflows/test_code_quality.yml
@@ -18,7 +18,7 @@ jobs:
         run: python3 -m pip install pre-commit
       - id: changed-files
         name: identify modified files
-        uses: tj-actions/changed-files@v40
+        uses: tj-actions/changed-files@v41
       - name: run pre-commit hooks on modified files
         run: pre-commit run --color always --files ${{ steps.changed-files.outputs.all_changed_files }} --show-diff-on-failure
       - name: check missing __init__ files
@@ -30,7 +30,8 @@ jobs:
     permissions:
       pull-requests: read
     outputs:
-      components: ${{ steps.filter.outputs.changes }}
+      datasets: ${{ steps.filter.outputs.datasets }}
+      pyproject: ${{ steps.filter.outputs.pyproject }}
     steps:
       - uses: dorny/paths-filter@v2
         id: filter
@@ -50,10 +51,14 @@ jobs:
     needs: code_quality
     name: run unit tests for base framework
     uses: ./.github/workflows/test_base.yml
-  unit_test_components:
+  unit_test_components_with_extras:
     needs: code_quality
-    name: run unit tests for individual components
-    uses: ./.github/workflows/test_components.yml
+    name: run unit tests for individual components with extras
+    uses: ./.github/workflows/test_components_with_extras.yml
+  unit_test_components_without_extras:
+    needs: code_quality
+    name: run unit tests for individual components without extras
+    uses: ./.github/workflows/test_components_without_extras.yml
   unit_test_datasets:
     needs: detect
     name: run unit tests for datasets
diff --git a/.github/workflows/test_components.yml b/.github/workflows/test_components.yml
deleted file mode 100644
index 33523721f..000000000
--- a/.github/workflows/test_components.yml
+++ /dev/null
@@ -1,135 +0,0 @@
-name: test modules
-on:
-  workflow_call:
-jobs:
-  detect:
-    name: detect change in module
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: read
-    outputs:
-      components: ${{ steps.filter.outputs.changes }}
-    steps:
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            alignment:
-              - sktime/base/**
-              - sktime/alignment/**
-            annotation:
-              - sktime/base/**
-              - sktime/annotation/**
-            classification:
-              - sktime/base/**
-              - sktime/classification/**
-            clustering:
-              - sktime/base/**
-              - sktime/clustering/**
-            forecasting:
-              - sktime/base/**
-              - sktime/forecasting/**
-            networks:
-              - sktime/base/**
-              - sktime/networks/**
-            param_est:
-              - sktime/base/**
-              - sktime/param_est/**
-            regression:
-              - sktime/base/**
-              - sktime/regression/**
-            transformations:
-              - sktime/base/**
-              - sktime/transformations/**
-  test:
-    needs: detect
-    name: test module
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        operating-system:
-          - macos-latest
-          - ubuntu-latest
-          - windows-latest
-        sktime-component: ${{ fromJSON(needs.detect_components_with_extras.outputs.components) }}
-    runs-on: ${{ matrix.operating-system }}
-    steps:
-      - name: checkout pull request branch
-        uses: actions/checkout@v4
-      - name: run tests for component ${{ matrix.sktime-component }} on python ${{ matrix.python-version }}
-        uses: ./.github/actions/test-component
-        with:
-          sktime-component-identifier: ${{ matrix.sktime-component }}
-          python-version-identifier: ${{ matrix.python-version }}
-          sub-sample-estimators: "True"
-          test-affected-estimators: "True"
-  detect_components_without_extras:
-    name: check for changes in individual components without extras
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: read
-    outputs:
-      components_without_extras: ${{ steps.filter.outputs.components_without_extras }}
-    steps:
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            components_without_extras:
-              - sktime/base/**
-              - sktime/_contrib/**
-              - sktime/benchmarking/**
-              - sktime/datatypes/**
-              - sktime/distances/**
-              - sktime/dists_kernels/**
-              - sktime/performance_metrics/**
-              - sktime/pipeline/**
-              - sktime/proba/**
-              - sktime/registry/**
-              - sktime/series_as_features/**
-              - sktime/split/**
-              - sktime/tests/**
-              - sktime/utils/**
-  test_components_without_extras:
-    needs: detect_components_without_extras
-    name: test all components without extras
-    if: ${{ needs.detect.outputs.components_without_extras == 'true' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        operating-system:
-          - macos-latest
-          - ubuntu-latest
-          - windows-latest
-    runs-on: ${{ matrix.operating-system }}
-    steps:
-      - name: checkout pull request branch
-        uses: actions/checkout@v4
-      - name: update local git tracking reference
-        run: git remote set-branches origin main
-      - name: update local shallow clone
-        run: git fetch --depth 1
-      - name: create python virtual environment
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: install core, test and all soft dependencies
-        run: python3 -m pip install .[all_extras_pandas2,tests]
-      - name: run unit tests
-        run: python3 -m pytest sktime --ignore sktime/alignment --ignore sktime/annotation --ignore sktime/classification --ignore sktime/clustering --ignore sktime/datasets --ignore sktime/forecasting --ignore sktime/networks --ignore sktime/param_est --ignore sktime/regression --ignore sktime/transformations --matrixdesign True --only_changed_modules True
-      - name: test coverage
-        uses: codecov/codecov-action@v3
-        with:
-          flags: ${{ matrix.python-version }},components_without_extras
diff --git a/.github/workflows/test_components_with_extras.yml b/.github/workflows/test_components_with_extras.yml
new file mode 100644
index 000000000..568e9b516
--- /dev/null
+++ b/.github/workflows/test_components_with_extras.yml
@@ -0,0 +1,75 @@
+name: test components with extras
+on:
+  workflow_call:
+jobs:
+  detect:
+    name: detect changes in components with extras
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      components_with_extras: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            alignment:
+              - sktime/base/**
+              - sktime/alignment/**
+            annotation:
+              - sktime/base/**
+              - sktime/annotation/**
+            classification:
+              - sktime/base/**
+              - sktime/classification/**
+            clustering:
+              - sktime/base/**
+              - sktime/clustering/**
+            forecasting:
+              - sktime/base/**
+              - sktime/forecasting/**
+            networks:
+              - sktime/base/**
+              - sktime/networks/**
+            param_est:
+              - sktime/base/**
+              - sktime/param_est/**
+            regression:
+              - sktime/base/**
+              - sktime/regression/**
+            transformations:
+              - sktime/base/**
+              - sktime/transformations/**
+  test:
+    needs: detect
+    name: test components with extras
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        operating-system:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+        sktime-component: ${{ fromJSON(needs.detect.outputs.components_with_extras) }}
+    runs-on: ${{ matrix.operating-system }}
+    steps:
+      - name: checkout pull request branch
+        uses: actions/checkout@v4
+      - name: run tests for component ${{ matrix.sktime-component }} on python ${{ matrix.python-version }}
+        uses: ./.github/actions/test-component
+        with:
+          sktime-component-identifier: ${{ matrix.sktime-component }}
+          python-version-identifier: ${{ matrix.python-version }}
+          sub-sample-estimators: "True"
+          test-affected-estimators: "True"
+      - name: upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},${{ matrix.sktime-component }},incomplete
diff --git a/.github/workflows/test_components_without_extras.yml b/.github/workflows/test_components_without_extras.yml
new file mode 100644
index 000000000..105a75759
--- /dev/null
+++ b/.github/workflows/test_components_without_extras.yml
@@ -0,0 +1,84 @@
+name: test components without extras
+on:
+  workflow_call:
+jobs:
+  detect:
+    name: detect changes in components without extras
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      components_without_extras: ${{ steps.filter.outputs.components_without_extras }}
+    steps:
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            components_without_extras:
+              - sktime/base/**
+              - sktime/_contrib/**
+              - sktime/benchmarking/**
+              - sktime/datatypes/**
+              - sktime/distances/**
+              - sktime/dists_kernels/**
+              - sktime/performance_metrics/**
+              - sktime/pipeline/**
+              - sktime/proba/**
+              - sktime/registry/**
+              - sktime/series_as_features/**
+              - sktime/split/**
+              - sktime/tests/**
+              - sktime/utils/**
+  test_components_without_extras:
+    needs: detect
+    name: test components without extras
+    if: ${{ needs.detect.outputs.components_without_extras == 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        operating-system:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+    runs-on: ${{ matrix.operating-system }}
+    steps:
+      - name: checkout pull request branch
+        uses: actions/checkout@v4
+      - name: update local git tracking reference
+        run: git remote set-branches origin main
+      - name: update local shallow clone
+        run: git fetch --depth 1
+      - name: create python virtual environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: install core, test and all soft dependencies
+        run: python3 -m pip install .[all_extras_pandas2,tests]
+      - name: run unit tests
+        run: >-
+          python3
+          -m pytest
+          sktime
+          --ignore sktime/base
+          --ignore sktime/datasets
+          --ignore sktime/alignment
+          --ignore sktime/annotation
+          --ignore sktime/classification
+          --ignore sktime/clustering
+          --ignore sktime/forecasting
+          --ignore sktime/networks
+          --ignore sktime/param_est
+          --ignore sktime/regression
+          --ignore sktime/transformations
+          --matrixdesign True
+          --only_changed_modules True
+      - name: upload coverage
+        uses: codecov/codecov-action@v3
+        with:
+          flags: ${{ matrix.operating-system }},${{ matrix.python-version }},components_without_extras,incomplete
diff --git a/sktime/registry/tests/test_scitype.py b/sktime/registry/tests/test_scitype.py
index 00aa58979..c3370a07c 100644
--- a/sktime/registry/tests/test_scitype.py
+++ b/sktime/registry/tests/test_scitype.py
@@ -35,3 +35,60 @@ def test_scitype(coerce_to_list):
         assert "transformer" == result_transformer[0]
     else:
         assert "transformer" == result_transformer
+
+
+@pytest.mark.parametrize("force_single_scitype", [True, False])
+@pytest.mark.parametrize("coerce_to_list", [True, False])
+def test_scitype_generic(force_single_scitype, coerce_to_list):
+    """Test that the scitype function recovers the correct scitype(s)."""
+    from sktime.base import BaseObject
+
+    class _DummyClass(BaseObject):
+        _tags = {"object_type": ["foo", "bar"]}
+
+    scitype_inferred = scitype(
+        _DummyClass(),
+        force_single_scitype=force_single_scitype,
+        coerce_to_list=coerce_to_list,
+    )
+
+    if force_single_scitype and coerce_to_list:
+        expected = ["foo"]
+    if not force_single_scitype and coerce_to_list:
+        expected = ["foo", "bar"]
+    if not coerce_to_list:
+        expected = "foo"
+
+    assert scitype_inferred == expected
+
+    class _DummyClass2(BaseObject):
+        _tags = {"object_type": "foo"}
+
+    scitype_inferred = scitype(
+        _DummyClass2(),
+        force_single_scitype=force_single_scitype,
+        coerce_to_list=coerce_to_list,
+    )
+
+    if coerce_to_list:
+        expected = ["foo"]
+    if not coerce_to_list:
+        expected = "foo"
+
+    assert scitype_inferred == expected
+
+    class _DummyClass3(BaseObject):
+        _tags = {"object_type": ["foo"]}
+
+    scitype_inferred = scitype(
+        _DummyClass3(),
+        force_single_scitype=force_single_scitype,
+        coerce_to_list=coerce_to_list,
+    )
+
+    if coerce_to_list:
+        expected = ["foo"]
+    if not coerce_to_list:
+        expected = "foo"
+
+    assert scitype_inferred == expected