Repository: sktime/sktime
Pull request #5756 (2024-01-16 01:44:06)

[DOC] update soft dependency handling guide for tests with tag based dependency checking
This updates the soft dependency handling guide for tests with current practice - which is class tag based depedency checking, as opposed to manually duplicating the soft dependency information.


Patch:
diff --git a/docs/source/developer_guide/dependencies.rst b/docs/source/developer_guide/dependencies.rst
index 92f4f3b4a..c08339849 100644
--- a/docs/source/developer_guide/dependencies.rst
+++ b/docs/source/developer_guide/dependencies.rst
@@ -52,11 +52,19 @@ Estimators with a soft dependency need to ensure the following:
    To do this, add a ``# doctest: +SKIP`` to the end of each line in the doctest to skip it entirely.
    See ``forecasting.arima.ARIMA`` as as an example. If concerned that skipping the test will reduce test coverage,
    consider exposing the doctest example as a pytest test function instead, see below how to handle soft dependencies in pytest functions.
-*  Decorate all ``pytest`` tests that import soft dependencies with a ``@pytest.mark.skipif(...)`` conditional on a check to ``_check_soft_dependencies``
-   for your new soft dependency, with ``severity="none"``.  Be sure that all soft dependencies imported for testing are imported within the test function itself,
-   rather than for the whole module!  This decorator will then skip your test, including imports, unless the system has the required packages installed.
-   This prevents crashes for any users running ``check_estimator`` on all estimators, or a full local ``pytest`` run without the required soft dependency.
-   See the tests in ``forecasting.tests.test_pmdarima`` for a concrete example.
+*  Decorate all ``pytest`` tests that import soft dependencies with a ``@pytest.mark.skipif(...)`` conditional on a soft dependency check.
+   If the test is specific to a single estimator or object, use ``run_test_for_class`` from ``sktime.tests.test_switch``
+   to mediate the condition through the class tags.
+   Otherwise, use ``_check_soft_dependencies`` for your new soft dependency, with ``severity="none"``.
+   Be sure that all soft dependencies imported for testing are imported within the test function itself,
+   rather than at root level (at the top) of the module.
+   This decorator will then skip your test, including imports,
+   unless the system has the required packages installed.
+   This prevents crashes for any users running ``check_estimator`` on all estimators,
+   or a full local ``pytest`` run without the required soft dependency.
+   See the tests in ``forecasting.tests.test_pmdarima`` for a concrete example of
+   ``run_test_for_class`` usage to decorate a test. See ``utils.tests.test_plotting``
+   for an example of ``_check_soft_dependencies`` usage.
 
 Adding and maintaining soft dependencies
 ----------------------------------------
diff --git a/sktime/performance_metrics/forecasting/_classes.py b/sktime/performance_metrics/forecasting/_classes.py
index 08d331383..e8311bd53 100644
--- a/sktime/performance_metrics/forecasting/_classes.py
+++ b/sktime/performance_metrics/forecasting/_classes.py
@@ -573,6 +573,10 @@ class BaseForecastingErrorMetricFunc(BaseForecastingErrorMetric):
         else:
             func = self.func
 
+        return self._evaluate_func(func=func, y_true=y_true, y_pred=y_pred, **params)
+
+    def _evaluate_func(self, func, y_true, y_pred, **params):
+        """Call func with kwargs subset to func parameters."""
         # import here for now to avoid interaction with getmembers in tests
         # todo: clean up ancient getmembers in test_metrics_classes
         from functools import partial
@@ -584,6 +588,15 @@ class BaseForecastingErrorMetricFunc(BaseForecastingErrorMetric):
             func_params = func_params.intersection(params.keys())
             params = {key: params[key] for key in func_params}
 
+        # deal with sklearn specific parameter constraints
+        # as these are a decorator, they obfuscate python native inspection
+        # via signature, so have to be dealt with separately
+        if hasattr(func, "_skl_parameter_constraints"):
+            constr = func._skl_parameter_constraints
+            if isinstance(constr, dict):
+                constr_params = set(constr.keys()).intersection(params.keys())
+                params = {key: params[key] for key in constr_params}
+
         res = func(y_true=y_true, y_pred=y_pred, **params)
         return res
 
@@ -609,6 +622,16 @@ class _DynamicForecastingErrorMetric(BaseForecastingErrorMetricFunc):
 
         self.set_tags(**{"lower_is_better": lower_is_better})
 
+    def _evaluate(self, y_true, y_pred, **kwargs):
+        """Evaluate the desired metric on given inputs."""
+        # this dict should contain all parameters
+        params = kwargs
+        params.update({"multioutput": self.multioutput, "multilevel": self.multilevel})
+
+        func = self.func
+
+        return self._evaluate_func(func=func, y_true=y_true, y_pred=y_pred, **params)
+
     @classmethod
     def get_test_params(cls, parameter_set="default"):
         """Return testing parameter settings for the estimator.


Test patch:
diff --git a/sktime/performance_metrics/forecasting/tests/test_metrics.py b/sktime/performance_metrics/forecasting/tests/test_metrics.py
index 1b2d07c4f..12f9709bf 100644
--- a/sktime/performance_metrics/forecasting/tests/test_metrics.py
+++ b/sktime/performance_metrics/forecasting/tests/test_metrics.py
@@ -129,3 +129,18 @@ def test_make_scorer():
     scorer = make_forecasting_scorer(rmsle, name="RMSLE")
 
     scorer.evaluate(pd.Series([1, 2, 3]), pd.Series([1, 2, 4]))
+
+
+def test_make_scorer_sklearn():
+    """Test make_forecasting_scorer and the failure case in #5715.
+
+    Naive adaptation fails on newer sklearn versions due to
+    decoration with sklearn's custom input constraint wrapper.
+    """
+    from sklearn.metrics import mean_absolute_error
+
+    from sktime.performance_metrics.forecasting import make_forecasting_scorer
+
+    scorer = make_forecasting_scorer(mean_absolute_error, name="RMSLE")
+
+    scorer.evaluate(pd.Series([1, 2, 3]), pd.Series([1, 2, 4]))