Repository: sktime/sktime
Pull request #6460 (2024-05-21 19:39:39)

[ENH] unsigned area under the calibration curve metric for distribution forecasts
This PR adds the unsigned area under the calibration curve as a metric for distribution forecasts.

FYI @benHeid, @astrogilda.


Patch:
diff --git a/.all-contributorsrc b/.all-contributorsrc
index 4d039c995..630f97689 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -2803,6 +2803,15 @@
       "contributions": [
         "maintenance"
       ]
+    },
+    {
+      "login": "helloplayer1",
+      "name": "Julian Haderlein",
+      "avatar_url": "https://avatars.githubusercontent.com/u/32032467?v=4",
+      "profile": "https://github.com/helloplayer1",
+      "contributions": [
+        "doc"
+      ]
     }
   ]
 }
diff --git a/docs/source/api_reference/forecasting.rst b/docs/source/api_reference/forecasting.rst
index 97cc002fe..1b03d86a6 100644
--- a/docs/source/api_reference/forecasting.rst
+++ b/docs/source/api_reference/forecasting.rst
@@ -388,6 +388,14 @@ Deep learning based forecasters
     NeuralForecastRNN
     NeuralForecastLSTM
 
+.. currentmodule:: sktime.forecasting.pykan_forecaster
+
+.. autosummary::
+    :toctree: auto_generated/
+    :template: class.rst
+
+    PyKANForecaster
+
 
 Intermittent time series forecasters
 ------------------------------------
diff --git a/docs/source/api_reference/performance_metrics.rst b/docs/source/api_reference/performance_metrics.rst
index 1a7a67c83..333ab2680 100644
--- a/docs/source/api_reference/performance_metrics.rst
+++ b/docs/source/api_reference/performance_metrics.rst
@@ -98,6 +98,7 @@ Distribution forecasts
     :toctree: auto_generated/
     :template: class_with_call.rst
 
+    AUCalibration
     CRPS
     LogLoss
     SquaredDistrLoss
diff --git a/examples/02a_classification_multivariate_cnn.ipynb b/examples/02a_classification_multivariate_cnn.ipynb
deleted file mode 100644
index f4b464240..000000000
--- a/examples/02a_classification_multivariate_cnn.ipynb
+++ /dev/null
@@ -1,136 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Multi-variate time series classification using a simple CNN\n",
-    "\n",
-    "In this notebook, we use sktime to perform for multi-variate time series classification by deep learning."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import seaborn as sns\n",
-    "from sklearn.model_selection import GridSearchCV\n",
-    "\n",
-    "from sktime.classification.deep_learning.cnn import CNNClassifier\n",
-    "from sktime.datasets import load_basic_motions\n",
-    "\n",
-    "sns.set_style(\"whitegrid\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Load a dataset\n",
-    "The Basic Motions dataset, from [timeseriesclassification.com](http://www.timeseriesclassification.com), has time series in six dimensions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "X_train, y_train = load_basic_motions(split=\"train\", return_X_y=True)\n",
-    "X_test, y_test = load_basic_motions(split=\"test\", return_X_y=True)\n",
-    "print(X_train.shape)\n",
-    "print(X_test.shape)\n",
-    "print(type(X_train.iloc[1, 1]))\n",
-    "X_train.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# The class labels\n",
-    "np.unique(y_train)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Train a deep neural network classifier\n",
-    "Here we choose to use the CNN (convolutional neural network) classifier. Other classifiers provided by sktime-dl include MLP, ResNet, InceptionTime and MCDCNN (Multi Channel Deep Convolutional Neural Network)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "network = CNNClassifier(n_epochs=50, verbose=True)\n",
-    "network.fit(X_train, y_train)\n",
-    "network.score(X_test, y_test)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Grid Search\n",
-    "sktime-dl is compatible with scikit-learn and can use sklearn's GridSearchCV.\n",
-    "\n",
-    "Here we search over two parameters, CNN kernel size and pooling size."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "param_grid = {\"kernel_size\": [7, 9], \"avg_pool_size\": [3, 5]}\n",
-    "grid = GridSearchCV(network, param_grid=param_grid, cv=3)\n",
-    "grid.fit(X_train, y_train)\n",
-    "\n",
-    "print(\"Best cross-validation accuracy: {:.2f}\".format(grid.best_score_))\n",
-    "print(\"Test set score: {:.2f}\".format(grid.score(X_test, y_test)))\n",
-    "print(\"Best parameters: {}\".format(grid.best_params_))"
-   ]
-  }
- ],
- "metadata": {
-  "file_extension": ".py",
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.5"
-  },
-  "mimetype": "text/x-python",
-  "name": "python",
-  "npconvert_exporter": "python",
-  "pygments_lexer": "ipython3",
-  "version": 3
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/examples/02b_classification_multivariate_lstmfcn.ipynb b/examples/02b_classification_multivariate_lstmfcn.ipynb
deleted file mode 100644
index f31975c52..000000000
--- a/examples/02b_classification_multivariate_lstmfcn.ipynb
+++ /dev/null
@@ -1,116 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Multi-variate time series classification using a simple CNN\n",
-    "\n",
-    "In this notebook, we use sktime to perform for multi-variate time series classification by deep learning.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import seaborn as sns\n",
-    "\n",
-    "from sktime.classification.deep_learning.lstmfcn import LSTMFCNClassifier\n",
-    "from sktime.datasets import load_basic_motions\n",
-    "\n",
-    "sns.set_style(\"whitegrid\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Load a dataset\n",
-    "\n",
-    "The Basic Motions dataset, from [timeseriesclassification.com](http://www.timeseriesclassification.com), has time series in six dimensions.\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "X_train, y_train = load_basic_motions(split=\"train\", return_X_y=True)\n",
-    "X_test, y_test = load_basic_motions(split=\"test\", return_X_y=True)\n",
-    "print(X_train.shape)\n",
-    "print(X_test.shape)\n",
-    "print(type(X_train.iloc[1, 1]))\n",
-    "X_train.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# The class labels\n",
-    "np.unique(y_train)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Train a LSTM-FCN\n",
-    "\n",
-    "In this exampe we use a LSTM-FCN (LongShort Term Memory Fully Convolutional Network) classifier originally implemented in sktime-dl.\n",
-    "Other classifiers provided by sktime-dl include CNN, MLP, ResNet, InceptionTime and MCDCNN.\n",
-    "\n",
-    "The LSTM-FCN estimator is compatible with scikit-learn and can use sklearn's GridSearchCV."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "network = LSTMFCNClassifier(n_epochs=65, verbose=0)\n",
-    "network.fit(X_train, y_train)\n",
-    "network.score(X_test, y_test)"
-   ]
-  }
- ],
- "metadata": {
-  "file_extension": ".py",
-  "kernelspec": {
-   "display_name": "Python 3.9.6 64-bit ('sktime')",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.6"
-  },
-  "mimetype": "text/x-python",
-  "name": "python",
-  "npconvert_exporter": "python",
-  "pygments_lexer": "ipython3",
-  "version": 3,
-  "vscode": {
-   "interpreter": {
-    "hash": "b1429b21f9b0cdf277ed408d65e56e85a8ed0b1a3006279bd7cbcc8cf36ab395"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/examples/02c_classification_multivariate_inceptiontime.ipynb b/examples/02c_classification_multivariate_inceptiontime.ipynb
deleted file mode 100644
index 6094630fd..000000000
--- a/examples/02c_classification_multivariate_inceptiontime.ipynb
+++ /dev/null
@@ -1,153 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Multi-variate time series classification using InceptionTime\n",
-    "\n",
-    "In this notebook, we use InceptionTime to perform for multi-variate time series classification by deep learning."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "\"\"\" Imports \"\"\"\n",
-    "import numpy as np\n",
-    "import seaborn as sns\n",
-    "from sklearn.model_selection import GridSearchCV\n",
-    "\n",
-    "from sktime.classification.deep_learning.inceptiontime import InceptionTimeClassifier\n",
-    "from sktime.datasets import load_basic_motions\n",
-    "\n",
-    "sns.set_style(\"whitegrid\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Load a dataset\n",
-    "The Basic Motions dataset, from [timeseriesclassification.com](http://www.timeseriesclassification.com), has time series in six dimensions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "X_train, y_train = load_basic_motions(split=\"train\", return_X_y=True)\n",
-    "X_test, y_test = load_basic_motions(split=\"test\", return_X_y=True)\n",
-    "print(X_train.shape)\n",
-    "print(X_test.shape)\n",
-    "X_train.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "# The class labels\n",
-    "np.unique(y_train)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Train a deep neural network classifier\n",
-    "Here we choose to use the InceptionTime classifier. Other classifiers provided by sktime-dl include MLP, ResNet, MCDCNN (Multi Channel Deep Convolutional Neural Network)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "network = InceptionTimeClassifier(n_epochs=75, verbose=False)\n",
-    "network.fit(X_train, y_train)\n",
-    "network.score(X_test, y_test)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Grid Search\n",
-    "sktime-dl is compatible with scikit-learn and can use sklearn's GridSearchCV.\n",
-    "\n",
-    "Here we search over two parameters, CNN kernel size and pooling size."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "pycharm": {
-     "is_executing": true
-    }
-   },
-   "outputs": [],
-   "source": [
-    "param_grid = {\"nb_filters\": [16, 32], \"bottleneck_size\": [16, 32]}\n",
-    "grid = GridSearchCV(network, param_grid=param_grid, cv=5)\n",
-    "# commented fit call to pass checks (meet time constraints)\n",
-    "# grid.fit(X_train, y_train)\n",
-    "\n",
-    "# print(\"Best cross-validation accuracy: {:.2f}\".format(grid.best_score_))\n",
-    "# print(\"Test set score: {:.2f}\".format(grid.score(X_test, y_test)))\n",
-    "# print(\"Best parameters: {}\".format(grid.best_params_))"
-   ]
-  }
- ],
- "metadata": {
-  "file_extension": ".py",
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.5"
-  },
-  "mimetype": "text/x-python",
-  "name": "python",
-  "npconvert_exporter": "python",
-  "pygments_lexer": "ipython3",
-  "version": 3
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/examples/AA_datatypes_and_datasets.ipynb b/examples/AA_datatypes_and_datasets.ipynb
index acff86913..adb440011 100644
--- a/examples/AA_datatypes_and_datasets.ipynb
+++ b/examples/AA_datatypes_and_datasets.ipynb
@@ -339,7 +339,7 @@
     "* variable names: the `\"np.ndarray\"` mtype cannot represent variable names.\n",
     "* time points: the rows of `obj` correspond to different, distinct time points. \n",
     "* time index: The time index is implicit and by-convention. The `i`-th row (for an integer `i`) is interpreted as an observation at the time point `i`.\n",
-    "* capabilities: cannot represent multivariate series; cannot represent unequally spaced series"
+    "* capabilities: can represent multivariate series; cannot represent unequally spaced series"
    ]
   },
   {
diff --git a/pyproject.toml b/pyproject.toml
index ae8a7e8fb..124f82231 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -244,7 +244,7 @@ docs = [
   "pydata-sphinx-theme",
   "Sphinx!=7.2.0,<8.0.0",
   "sphinx-copybutton",
-  "sphinx-design<0.6.0",
+  "sphinx-design<0.7.0",
   "sphinx-gallery<0.17.0",
   "sphinx-issues<5.0.0",
   "tabulate",
@@ -274,6 +274,7 @@ dl = [
   'tensorflow<2.17,>=2; python_version < "3.12"',
   'torch; python_version < "3.12"',
   'transformers[torch]<4.41.0; python_version < "3.12"',
+  'pykan; python_version > "3.9.7"'
 ]
 mlflow = [
   "mlflow",
diff --git a/sktime/classification/base.py b/sktime/classification/base.py
index cca5c0e8e..4f09ca6da 100644
--- a/sktime/classification/base.py
+++ b/sktime/classification/base.py
@@ -727,6 +727,9 @@ class BaseClassifier(BasePanelMixin):
         y_proba = self._predict_proba(X)
         y_pred = y_proba.argmax(axis=1)
 
+        # restore class labels if they were not originally integer
+        y_pred = self.classes_[y_pred]
+
         return y_pred
 
     def _predict_proba(self, X) -> np.ndarray:
diff --git a/sktime/classification/dictionary_based/_boss.py b/sktime/classification/dictionary_based/_boss.py
index 0bcb4def2..0c1b234b1 100644
--- a/sktime/classification/dictionary_based/_boss.py
+++ b/sktime/classification/dictionary_based/_boss.py
@@ -639,9 +639,7 @@ class IndividualBOSS(BaseClassifier):
             Predicted class labels.
         """
         test_bags = self._transformer.transform(X)
-        data_type = type(self._class_vals[0])
-        if data_type == np.str_ or data_type == str:
-            data_type = "object"
+        data_type = self._class_vals.dtype
 
         classes = np.zeros(test_bags.shape[0], dtype=data_type)
 
diff --git a/sktime/classification/distance_based/_proximity_forest.py b/sktime/classification/distance_based/_proximity_forest.py
index 1626b7e42..eb35e836e 100644
--- a/sktime/classification/distance_based/_proximity_forest.py
+++ b/sktime/classification/distance_based/_proximity_forest.py
@@ -900,33 +900,6 @@ class ProximityStump(BaseClassifier):
         self.entropy = gini_gain(self.y, self.y_branches)
         return self
 
-    def _predict(self, X) -> np.ndarray:
-        """Predicts labels for sequences in X.
-
-        Parameters
-        ----------
-        X : array-like or sparse matrix of shape = [n_instances, n_columns]
-            The training input samples.
-            If a Pandas data frame is passed (sktime format)
-            If a Pandas data frame is passed, a check is performed that it
-            only has one column.
-            If not, an exception is thrown, since this classifier does not
-            yet have
-            multivariate capability.
-
-        Returns
-        -------
-        y : array-like, shape =  [n_instances] - predicted class labels
-        """
-        distributions = self._predict_proba(X)
-        predictions = []
-        for instance_index in range(0, X.shape[0]):
-            distribution = distributions[instance_index]
-            prediction = np.argmax(distribution)
-            predictions.append(prediction)
-
-        return np.array(predictions)
-
     def _predict_proba(self, X) -> np.ndarray:
         """Find probability estimates for each class for all cases in X.
 
@@ -1118,33 +1091,6 @@ class ProximityTree(BaseClassifier):
 
         return self
 
-    def _predict(self, X) -> np.ndarray:
-        """Predicts labels for sequences in X.
-
-        Parameters
-        ----------
-        X : array-like or sparse matrix of shape = [n_instances, n_columns]
-            The training input samples.
-            If a Pandas data frame is passed (sktime format)
-            If a Pandas data frame is passed, a check is performed that it
-            only has one column.
-            If not, an exception is thrown, since this classifier does not
-            yet have
-            multivariate capability.
-
-        Returns
-        -------
-        y : array-like, shape =  [n_instances] - predicted class labels
-        """
-        distributions = self._predict_proba(X)
-        predictions = []
-        for instance_index in range(0, X.shape[0]):
-            distribution = distributions[instance_index]
-            prediction = np.argmax(distribution)
-            predictions.append(prediction)
-
-        return np.array(predictions)
-
     def _predict_proba(self, X) -> np.ndarray:
         """Find probability estimates for each class for all cases in X.
 
@@ -1494,33 +1440,6 @@ class ProximityForest(BaseClassifier):
         """
         return tree.predict_proba(X)
 
-    def _predict(self, X) -> np.ndarray:
-        """Predicts labels for sequences in X.
-
-        Parameters
-        ----------
-        X : array-like or sparse matrix of shape = [n_instances, n_columns]
-            The training input samples.
-            If a Pandas data frame is passed (sktime format)
-            If a Pandas data frame is passed, a check is performed that it
-            only has one column.
-            If not, an exception is thrown, since this classifier does not
-            yet have
-            multivariate capability.
-
-        Returns
-        -------
-        y : array-like, shape =  [n_instances] - predicted class labels
-        """
-        distributions = self._predict_proba(X)
-        predictions = []
-        for instance_index in range(0, X.shape[0]):
-            distribution = distributions[instance_index]
-            prediction = np.argmax(distribution)
-            predictions.append(prediction)
-
-        return np.array(predictions)
-
     def _predict_proba(self, X) -> np.ndarray:
         """Find probability estimates for each class for all cases in X.
 
diff --git a/sktime/classification/ensemble/_bagging.py b/sktime/classification/ensemble/_bagging.py
index 17bbcf391..ccb55f30c 100644
--- a/sktime/classification/ensemble/_bagging.py
+++ b/sktime/classification/ensemble/_bagging.py
@@ -163,6 +163,7 @@ class BaggingClassifier(BaseClassifier):
             n_features_ = n_features
 
         self.estimators_ = []
+        self._col_ixis = []
         for _i in range(n_estimators):
             esti = estimator.clone()
             row_iloc = pd.RangeIndex(n)
@@ -183,6 +184,7 @@ class BaggingClassifier(BaseClassifier):
 
             yi = y[row_ss]
             self.estimators_ += [esti.fit(Xi, yi)]
+            self._col_ixis += [col_ix_i]
 
         return self
 
@@ -200,7 +202,14 @@ class BaggingClassifier(BaseClassifier):
             Predicted probabilities using the ordering in classes_.
         """
         classes = pd.Index(self.classes_)
-        y_probas = [est.predict_proba(X) for est in self.estimators_]
+
+        y_probas = []
+        for esti, col_ix_i in zip(self.estimators_, self._col_ixis):
+            Xi = X.loc[:, col_ix_i]
+            if self.bootstrap_features:
+                Xi.columns = pd.RangeIndex(len(col_ix_i))
+
+            y_probas += [esti.predict_proba(Xi)]
 
         est_shape = (len(y_probas[0]), len(classes))
         y_proba_np = np.zeros((len(y_probas), est_shape[0], est_shape[1]))
@@ -238,16 +247,16 @@ class BaggingClassifier(BaseClassifier):
             instance.
             ``create_test_instance`` uses the first (or only) dictionary in ``params``.
         """
-        from sktime.classification.dummy import DummyClassifier
+        from sktime.classification.feature_based import SummaryClassifier
 
-        params1 = {"estimator": DummyClassifier()}
+        params1 = {"estimator": SummaryClassifier()}
         params2 = {
-            "estimator": DummyClassifier(),
+            "estimator": SummaryClassifier(),
             "n_samples": 0.5,
             "n_features": 0.5,
         }
         params3 = {
-            "estimator": DummyClassifier(),
+            "estimator": SummaryClassifier(),
             "n_samples": 7,
             "n_features": 2,
             "bootstrap": False,
@@ -255,7 +264,7 @@ class BaggingClassifier(BaseClassifier):
         }
 
         # force-create a classifier that cannot handle multivariate
-        univariate_dummy = DummyClassifier()
+        univariate_dummy = SummaryClassifier()
         univariate_dummy.set_tags(**{"capability:multivariate": False})
         # this should still result in a multivariate classifier
         params4 = {
diff --git a/sktime/forecasting/compose/_pipeline.py b/sktime/forecasting/compose/_pipeline.py
index 0338e3615..8d372f6dd 100644
--- a/sktime/forecasting/compose/_pipeline.py
+++ b/sktime/forecasting/compose/_pipeline.py
@@ -1376,13 +1376,13 @@ class ForecastX(BaseForecaster):
         self.fh_X = fh_X
         self.behaviour = behaviour
         self.columns = columns
-        self.forecaster_X_exogeneous = forecaster_X_exogeneous
         if isinstance(forecaster_X_exogeneous, str):
             if forecaster_X_exogeneous not in ["None", "complement"]:
                 raise ValueError(
                     'forecaster_X_exogeneous must be one of "None", "complement",'
                     "or a pandas.Index coercible"
                 )
+        self.forecaster_X_exogeneous = forecaster_X_exogeneous
 
         if predict_behaviour not in ["use_forecasts", "use_actuals"]:
             raise ValueError(
@@ -1583,9 +1583,11 @@ class ForecastX(BaseForecaster):
             return None
 
         if ixx == "complement":
-            X_for_fcX = X.drop(columns=self.columns)
+            X_for_fcX = X.drop(columns=self.columns, errors="ignore")
+
             if X_for_fcX.shape[1] < 1:
                 return None
+
             return X_for_fcX
 
         ixx_pd = pd.Index(ixx)
diff --git a/sktime/forecasting/pykan_forecaster.py b/sktime/forecasting/pykan_forecaster.py
new file mode 100644
index 000000000..a84ff789e
--- /dev/null
+++ b/sktime/forecasting/pykan_forecaster.py
@@ -0,0 +1,310 @@
+# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
+"""Dummy forecasters."""
+
+__author__ = ["bheidri"]
+
+import pandas as pd
+from skbase.utils.dependencies import _check_soft_dependencies
+
+from sktime.forecasting.base import BaseForecaster
+from sktime.split import temporal_train_test_split
+
+if _check_soft_dependencies("torch", severity="none"):
+    import torch
+    from torch.utils.data import Dataset
+else:
+
+    class Dataset:
+        """Dummy class if torch is unavailable."""
+
+        pass
+
+
+if _check_soft_dependencies("kan", severity="none"):
+    from kan import KAN
+
+
+class PyKANForecaster(BaseForecaster):
+    """
+    PyKANForecaster uses Kolmogorov Arnold Network [1] to forecast time series data.
+
+    This forecaster uses the pykan library to create a KAN model to forecast
+    time series data. The model is trained on the training data and then used to
+    forecast the future values of the time series.
+    **Note** This forecaster is experimental and the used library for implementing
+    the KANs may be exchanged in the future to a more stable and efficient library.
+
+    Parameters
+    ----------
+    hidden_layers : tuple, optional (default=(5,5))
+        The number of hidden layers in the network.
+    input_layer_size : int, optional (default=12)
+        The size of the input layer.
+    k : int, optional (default=3)
+        The number of nearest neighbors to consider.
+    grids : np.array, optional (default=np.array([5,10,20, 50, 100]))
+        The grid sizes to use in the model.
+    model_params : dict, optional (default=None)
+        The parameters to pass to the model. See pykan documentation for more details.
+    fit_params : dict, optional (default=None)
+        The parameters to pass to the fit method. See pykan documentation
+        for more details.
+    val_size : float, optional (default=0.5)
+        The size of the validation set to use in the training.
+    device : str, optional (default="cpu")
+        The device to use for training the model.
+
+    References
+    ----------
+    .. [1] Liu, Ziming, et al. "KAN: Kolmogorov-Arnold Networks."
+      arXiv preprint arXiv:2404.19756 (2024).
+    """
+
+    _tags = {
+        "y_inner_mtype": "pd.Series",
+        "X_inner_mtype": "pd.DataFrame",
+        "scitype:y": "univariate",
+        "ignores-exogeneous-X": False,
+        "requires-fh-in-fit": True,
+        "X-y-must-have-same-index": True,
+        "enforce_index_type": None,
+        "handles-missing-data": False,
+        "capability:pred_int": False,
+        "capability:pred_int:insample": False,
+        "capability:insample": False,
+        "python_version": None,
+        "python_dependencies": ["kan", "torch"],
+    }
+
+    def __init__(
+        self,
+        hidden_layers=(5, 5),
+        input_layer_size=12,
+        grids=None,
+        model_params=None,
+        fit_params=None,
+        val_size=0.5,
+        device="cpu",
+    ):
+        self.hidden_layers = hidden_layers
+        self.input_layer_size = input_layer_size
+        self.grids = grids
+        self._grids = grids if grids is not None else [5, 10, 20, 50, 100]
+        self.val_size = val_size
+        self.model_params = model_params
+        self._model_params = model_params if model_params is not None else {}
+        self.fit_params = fit_params
+        self._fit_params = fit_params if fit_params is not None else {}
+        self.device = device
+        super().__init__()
+
+    def _fit(self, y, X, fh):
+        """Fit forecaster to training data.
+
+        private _fit containing the core logic, called from fit
+
+        Writes to self:
+            Sets fitted model attributes ending in "_".
+
+        Parameters
+        ----------
+        y : sktime time series object
+            guaranteed to be of an mtype in self.get_tag("y_inner_mtype")
+            Time series to which to fit the forecaster.
+            if self.get_tag("scitype:y")=="univariate":
+                guaranteed to have a single column/variable
+            if self.get_tag("scitype:y")=="multivariate":
+                guaranteed to have 2 or more columns
+            if self.get_tag("scitype:y")=="both": no restrictions apply
+        fh : guaranteed to be ForecastingHorizon or None, optional (default=None)
+            The forecasting horizon with the steps ahead to to predict.
+            guaranteed to be passed in _predict
+        X :  sktime time series object, optional (default=None)
+            guaranteed to be of an mtype in self.get_tag("X_inner_mtype")
+            Exogeneous time series to fit to.
+
+        Returns
+        -------
+        self : reference to self
+        """
+        output_size = max(fh.to_relative(self.cutoff)._values)
+        if X is not None:
+            y_train, y_test, X_train, X_test = temporal_train_test_split(
+                y, X=X, test_size=(self.val_size)
+            )
+            ds_train = PyTorchTrainDataset(
+                y_train, self.input_layer_size, output_size, X=X_train
+            )
+            ds_test = PyTorchTrainDataset(
+                y_test, self.input_layer_size, output_size, X=X_test
+            )
+            input_layer_size = self.input_layer_size + X_train.shape[1] * output_size
+        else:
+            y_train, y_test = temporal_train_test_split(y, test_size=(self.val_size))
+            ds_train = PyTorchTrainDataset(y_train, self.input_layer_size, output_size)
+            ds_test = PyTorchTrainDataset(y_test, self.input_layer_size, output_size)
+            input_layer_size = self.input_layer_size
+        train_input = [ds_train[i][0] for i in range(len(ds_train))]
+        train_target = [ds_train[i][1] for i in range(len(ds_train))]
+        test_input = [ds_test[i][0] for i in range(len(ds_test))]
+        test_target = [ds_test[i][1] for i in range(len(ds_test))]
+        # no fitting, we already know the forecast values
+
+        ds_new = {
+            "train_input": torch.stack(train_input).type(torch.float32).to(self.device),
+            "train_label": torch.stack(train_target)
+            .type(torch.float32)
+            .to(self.device),
+            "test_input": torch.stack(test_input).type(torch.float32).to(self.device),
+            "test_label": torch.stack(test_target).type(torch.float32).to(self.device),
+        }
+
+        self.train_losses = []
+        self.test_losses = []
+
+        self._layer_sizes = [input_layer_size, *self.hidden_layers, output_size]
+        for i in range(len(self._grids)):
+            if i == 0:
+                model = KAN(
+                    width=self._layer_sizes,
+                    grid=self.grids[i],
+                    device=self.device,
+                    **self._model_params,
+                )
+            if i != 0:
+                model = KAN(
+                    width=self._layer_sizes,
+                    grid=self._grids[i],
+                    device=self.device,
+                    **self._model_params,
+                ).initialize_from_another_model(model, ds_new["train_input"])
+            results = model.train(ds_new, device=self.device, **self._fit_params)
+            if len(self.test_losses) == 0 or results["test_loss"][-1] < min(
+                self.test_losses
+            ):
+                self._state_dict = model.state_dict()
+                self._best_grid = self._grids[i]
+            self.train_losses += results["train_loss"]
+            self.test_losses += results["test_loss"]
+        # self.state_dict = best_model.state_dict()
+        return self
+
+    def _predict(self, fh, X):
+        """Forecast time series at future horizon.
+
+        private _predict containing the core logic, called from predict
+
+        State required:
+            Requires state to be "fitted".
+
+        Accesses in self:
+            Fitted model attributes ending in "_"
+            self.cutoff
+
+        Parameters
+        ----------
+        fh : guaranteed to be ForecastingHorizon or None, optional (default=None)
+            The forecasting horizon with the steps ahead to to predict.
+        X : sktime time series object, optional (default=None)
+            guaranteed to be of an mtype in self.get_tag("X_inner_mtype")
+            Exogeneous time series for the forecast
+
+        Returns
+        -------
+        y_pred : sktime time series object
+            should be of the same type as seen in _fit, as in "y_inner_mtype" tag
+            Point predictions
+        """
+        model = KAN(width=self._layer_sizes, grid=self._best_grid, **self._model_params)
+        model.load_state_dict(self._state_dict)
+        if X is not None:
+            input_ = torch.cat(
+                [
+                    torch.from_numpy(self._y.values[-self.input_layer_size :]).reshape(
+                        (1, -1)
+                    ),
+                    torch.from_numpy(X.values).reshape((1, -1)),
+                ],
+                dim=-1,
+            )
+        else:
+            input_ = torch.from_numpy(self._y.values[-self.input_layer_size :]).reshape(
+                (1, -1)
+            )
+
+        prediction = model(input_).detach().numpy().reshape((-1,))
+        index = list(fh.to_absolute(self.cutoff))
+        return pd.Series(
+            prediction[fh.to_relative(self._cutoff) - 1], index=index, name=self._y.name
+        )
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the estimator.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return `"default"` set.
+            There are currently no reserved values for forecasters.
+
+        Returns
+        -------
+        params : dict or list of dict, default = {}
+            Parameters to create testing instances of the class
+            Each dict are parameters to construct an "interesting" test instance, i.e.,
+            `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance.
+            `create_test_instance` uses the first (or only) dictionary in `params`
+        """
+        params = [
+            {
+                "grids": [2, 3],
+                "model_params": {"k": 2},
+                "fit_params": {"steps": 1},
+                "input_layer_size": 2,
+                "hidden_layers": (1,),
+            },
+            {
+                "input_layer_size": 2,
+                "grids": [3],
+                "model_params": {"k": 2},
+                "fit_params": {"steps": 1},
+                "hidden_layers": (1, 1),
+            },
+        ]
+        return params
+
+
+class PyTorchTrainDataset(Dataset):
+    """Dataset for use in sktime deep learning forecasters."""
+
+    def __init__(self, y, seq_len, fh=None, X=None):
+        self.y = y.values
+        self.X = X.values if X is not None else X
+        self.seq_len = seq_len
+        self.fh = fh
+
+    def __len__(self):
+        """Return length of dataset."""
+        return max(len(self.y) - self.seq_len - self.fh + 1, 0)
+
+    def __getitem__(self, i):
+        """Return data point."""
+        from torch import from_numpy, tensor
+
+        hist_y = tensor(self.y[i : i + self.seq_len]).type(torch.float32)
+        if self.X is not None:
+            exog_data = (
+                tensor(self.X[i + self.seq_len : i + self.seq_len + self.fh])
+                .type(torch.float32)
+                .flatten()
+            )
+        else:
+            exog_data = tensor([])
+        return (
+            torch.cat([hist_y, exog_data]),
+            from_numpy(self.y[i + self.seq_len : i + self.seq_len + self.fh]).type(
+                torch.float32
+            ),
+        )
diff --git a/sktime/performance_metrics/forecasting/probabilistic/__init__.py b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
index fbab9f8df..68a072958 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/__init__.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/__init__.py
@@ -20,10 +20,12 @@ __all__ = [
     "IntervalWidth",
     "LogLoss",
     "SquaredDistrLoss",
+    "AUCalibration",
 ]
 
 from sktime.performance_metrics.forecasting.probabilistic._classes import (
     CRPS,
+    AUCalibration,
     ConstraintViolation,
     EmpiricalCoverage,
     IntervalWidth,
diff --git a/sktime/performance_metrics/forecasting/probabilistic/_classes.py b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
index cfee5b76f..95342e610 100644
--- a/sktime/performance_metrics/forecasting/probabilistic/_classes.py
+++ b/sktime/performance_metrics/forecasting/probabilistic/_classes.py
@@ -946,7 +946,7 @@ class CRPS(_BaseDistrForecastingMetric):
     multivariate : bool, optional, default=False
         if True, behaves as multivariate CRPS (sum of scores)
         CRPS is computed for entire row, results one score per row
-        if False, is univariate log-loss, per variable
+        if False, returns univariate CRPS, per variable
         CRPS is computed per variable marginal, results in many scores per row
     """  # noqa: E501
 
@@ -964,3 +964,73 @@ class CRPS(_BaseDistrForecastingMetric):
         params1 = {}
         params2 = {"multivariate": True}
         return [params1, params2]
+
+
+class AUCalibration(_BaseDistrForecastingMetric):
+    r"""Area under the calibration curve for distributional predictions.
+
+    Computes the unsigned area between the calibration curve and the diagonal.
+
+    The calibration curve is the cumulative curve of the sample of
+    predictive cumulative distibution functions evaluated at the true values.
+
+    Mathematically, let :math:`d_1, \dots, d_N` be the predictive distributions,
+    let :math:`y_1, \dots, y_N` be the true values, and let :math:`F_i` be the
+    cumulative distribution function of :math:`d_i`.
+
+    Define the calibration sample as :math:`c_i := F_i(y_i)`, for
+    :math:`i = 1, \dots, N`. For perfect predictions, the sample of :math:`c_i` will be
+    uniformly distributed on [0, 1], and i.i.d. from that uniform distribution.
+
+    Let :math:`c_{(i)}` be the :math:`i`-th order statistic of the sample of
+    :math:`c_i`, i.e., the :math:`i`-th smallest value in the sample.
+
+    The (unsigned) area under the calibration curve - or, more precisely,
+    between the diagonal and the calibration curve - is defined as
+
+    .. math:: \frac{1}{N} \sum_{i=1}^N \left| c_{(i)} - \frac{i}{N} \right|.
+
+    * ``evaluate`` returns the unsigned area between the calibration curve
+      and the diagonal, i.e., the above quantity.
+    * ``evaluate_by_index`` returns, for the :math:`i`-th test sample, the value
+      :math:`\left| c_i - \frac{r_i}{N} \right|`,
+      where :math:`r_i` is the rank of :math:`c_i`
+      in the sample of :math:`c_i`. In case of ties, tied ranks are averaged.
+    * ``multivariate`` controls averaging over variables.
+
+    Parameters
+    ----------
+    multioutput : {'raw_values', 'uniform_average'} or array-like of shape \
+            (n_outputs,), default='uniform_average'
+        Defines whether and how to aggregate metric for across variables.
+        If 'uniform_average' (default), errors are mean-averaged across variables.
+        If array-like, errors are weighted averaged across variables, values as weights.
+        If 'raw_values', does not average errors across variables, columns are retained.
+    multivariate : bool, optional, default=False
+        if True, behaves as multivariate metric (sum of scores)
+        The metric is computed for entire row, results one score per row
+        if False, returns univariate metric, per variable
+        The metric is computed per variable marginal, results in many scores per row
+    """  # noqa: E501
+
+    def __init__(self, multioutput="uniform_average", multivariate=False):
+        self.multivariate = multivariate
+        super().__init__(multioutput=multioutput)
+
+    def _evaluate_by_index(self, y_true, y_pred, multioutput, **kwargs):
+        cdfs = y_pred.cdf(y_true)
+        # using the average in case of ranks is fine
+        # because the absolute sums in the metric average out
+        cdfs_ranked = cdfs.rank(axis=0, method="average", pct=True)
+        n = cdfs.shape[0]
+        diagonal = np.arange(1, n + 1).reshape(-1, 1) / n
+
+        res = (cdfs_ranked - diagonal).abs()
+        return res
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Retrieve test parameters."""
+        params1 = {}
+        params2 = {"multivariate": True}
+        return [params1, params2]
diff --git a/sktime/utils/git_diff.py b/sktime/utils/git_diff.py
index f5cf6ef72..0d53af9ca 100644
--- a/sktime/utils/git_diff.py
+++ b/sktime/utils/git_diff.py
@@ -126,7 +126,6 @@ def get_changed_lines(file_path, only_indented=True):
         return []
 
 
-@lru_cache
 def get_packages_with_changed_specs():
     """Get packages with changed or added specs.
 
@@ -134,6 +133,20 @@ def get_packages_with_changed_specs():
     -------
     list of str : names of packages with changed or added specs
     """
+    return list(_get_packages_with_changed_specs())
+
+
+@lru_cache
+def _get_packages_with_changed_specs():
+    """Get packages with changed or added specs.
+
+    Private version of get_packages_with_changed_specs,
+    to avoid side effects on the list return.
+
+    Returns
+    -------
+    tuple of str : names of packages with changed or added specs
+    """
     from packaging.requirements import Requirement
 
     changed_lines = get_changed_lines("pyproject.toml")
@@ -161,6 +174,6 @@ def get_packages_with_changed_specs():
         packages.append(pkg)
 
     # make unique
-    packages = list(set(packages))
+    packages = tuple(set(packages))
 
     return packages


Test patch:
diff --git a/docs/source/developer_guide/testing_framework.rst b/docs/source/developer_guide/testing_framework.rst
index 40ebc982e..7841b26f5 100644
--- a/docs/source/developer_guide/testing_framework.rst
+++ b/docs/source/developer_guide/testing_framework.rst
@@ -68,7 +68,8 @@ See below for more details, or the `pytest documentation on the topic`_.
 The ``sktime`` plug-in for ``pytest`` generates the tuples of fixture values for this.
 In the above example, we loop over the following fixtures lists:
 
-* ``estimator_instance`` over estimator instances, obtained from all ``sktime`` estimators via ``create_test_instances_and_names``
+* ``estimator_instance`` over estimator instances, obtained from all ``sktime`` estimators via ``create_test_instances_and_names``, which
+  constructs instances from parameter settings in estimator classes' ``get_test_params``.
 
 * ``scenario`` objects, which encodes data inputs and method call sequences to ``estimator_instance`` (explained in further detail below).
 
diff --git a/sktime/classification/dictionary_based/tests/test_boss.py b/sktime/classification/dictionary_based/tests/test_boss.py
index 1bba876ec..f803785b1 100644
--- a/sktime/classification/dictionary_based/tests/test_boss.py
+++ b/sktime/classification/dictionary_based/tests/test_boss.py
@@ -25,7 +25,7 @@ def dataset():
 @pytest.mark.parametrize(
     "new_class,expected_dtype",
     [
-        ({"1": "Class1", "2": "Class2"}, object),
+        ({"1": "Class1", "2": "Class2"}, "same"),
         ({"1": 1, "2": 2}, int),
         ({"1": 1.0, "2": 2.0}, float),
         ({"1": True, "2": False}, bool),
@@ -39,6 +39,9 @@ def test_individual_boss_classes(dataset, new_class, expected_dtype):
     # change class
     y_train = np.array([new_class[y] for y in y_train])
 
+    if expected_dtype == "same":
+        expected_dtype = y_train.dtype
+
     # train iboss and predict X_test
     iboss = IndividualBOSS()
     iboss.fit(X_train, y_train)
diff --git a/sktime/classification/tests/test_all_classifiers.py b/sktime/classification/tests/test_all_classifiers.py
index 173edc08a..cd16cdfb5 100644
--- a/sktime/classification/tests/test_all_classifiers.py
+++ b/sktime/classification/tests/test_all_classifiers.py
@@ -120,20 +120,17 @@ class TestAllClassifiers(ClassifierFixtureGenerator, QuickTester):
         # we only use the first estimator instance for testing
         classname = estimator_class.__name__
 
-        # retrieve expected predict_proba output, and skip test if not available
-        if classname in unit_test_proba.keys():
-            expected_probas = unit_test_proba[classname]
-        else:
-            # skip test if no expected probas are registered
-            return None
-
         # if numba is not installed, some estimators may still try to construct
         # numba dependenct estimators in results_comparison
         # if that is the case, we skip the test
+        if classname in unit_test_proba.keys():
+            parameter_set = "results_comparison"
+        else:
+            parameter_set = "default"
         try:
             # we only use the first estimator instance for testing
             estimator_instance = estimator_class.create_test_instance(
-                parameter_set="results_comparison"
+                parameter_set=parameter_set
             )
         except ModuleNotFoundError as e:
             if not _check_soft_dependencies("numba", severity="none"):
@@ -152,10 +149,18 @@ class TestAllClassifiers(ClassifierFixtureGenerator, QuickTester):
 
         # train classifier and predict probas
         estimator_instance.fit(X_train, y_train)
+
+        y_pred = estimator_instance.predict(X_test.iloc[indices])
+        assert y_pred.dtype == y_train.dtype
+        assert set(y_train).issuperset(set(y_pred))
+
         y_proba = estimator_instance.predict_proba(X_test.iloc[indices])
 
-        # assert probabilities are the same
-        _assert_array_almost_equal(y_proba, expected_probas, decimal=2)
+        # retrieve expected predict_proba output, and skip test if not available
+        if classname in unit_test_proba.keys():
+            expected_probas = unit_test_proba[classname]
+            # assert probabilities are the same
+            _assert_array_almost_equal(y_proba, expected_probas, decimal=2)
 
     def test_classifier_on_basic_motions(self, estimator_class):
         """Test classifier on basic motions data."""
diff --git a/sktime/forecasting/compose/tests/test_forecastx.py b/sktime/forecasting/compose/tests/test_forecastx.py
index 80d308d19..aec4eccbd 100644
--- a/sktime/forecasting/compose/tests/test_forecastx.py
+++ b/sktime/forecasting/compose/tests/test_forecastx.py
@@ -391,3 +391,41 @@ def test_use_of_passed_unknown_X(predict_behaviour_option: str) -> None:
             mock_predict.assert_called_once()
         elif predict_behaviour_option == "use_actuals":
             mock_predict.assert_not_called()
+
+
+@pytest.mark.skipif(
+    not _check_estimator_deps(ARIMA, severity="none"),
+    reason="skip test if required soft dependency is not available",
+)
+@pytest.mark.parametrize("cols_to_forecast", [["GNPDEFL", "GNP"], ["ARMED", "POP"]])
+def test_forecaster_X_exogeneous(cols_to_forecast):
+    """Test that ForecastX forecaster_X uses exogenous data as told by parameter."""
+    from sktime.forecasting.compose import ForecastX
+    from sktime.split import temporal_train_test_split
+
+    y, X = load_longley()
+
+    fh = [1, 2, 3, 4]
+    y_train, _, X_train, X_test = temporal_train_test_split(y, X, test_size=max(fh))
+
+    forecaster = ARIMA()
+    pipeline1 = ForecastX(
+        forecaster.clone(),
+        forecaster_X=forecaster.clone(),
+        columns=cols_to_forecast,
+        forecaster_X_exogeneous="complement",
+    )
+
+    pipeline1.fit(y_train, X=X_train, fh=fh)
+    y_pred1 = pipeline1.predict(X=X_test.drop(columns=cols_to_forecast))
+
+    pipeline2 = ForecastX(
+        forecaster.clone(),
+        forecaster_X=forecaster.clone(),
+        columns=cols_to_forecast,
+        forecaster_X_exogeneous="None",
+    )
+
+    pipeline2.fit(y_train, X=X_train, fh=fh)
+    y_pred2 = pipeline2.predict(X=X_test.drop(columns=cols_to_forecast))
+    np.testing.assert_array_equal(y_pred1.index, y_pred2.index)
diff --git a/sktime/tests/_config.py b/sktime/tests/_config.py
index c2160a9cc..aca49b5e2 100644
--- a/sktime/tests/_config.py
+++ b/sktime/tests/_config.py
@@ -228,6 +228,7 @@ EXCLUDED_TESTS = {
     "LTSFDLinearForecaster": ["test_predict_time_index_in_sample_full"],
     "LTSFNLinearForecaster": ["test_predict_time_index_in_sample_full"],
     "HFTransformersForecaster": ["test_predict_time_index_in_sample_full"],
+    "PyKANForecaster": ["test_predict_time_index_in_sample_full"],
     "WEASEL": ["test_multiprocessing_idempotent"],  # see 5658
     # StatsForecastMSTL is failing in probabistic forecasts, see #5703, #5920
     "StatsForecastMSTL": ["test_pred_int_tag"],