Repository: sktime/sktime

Issue:
[BUG] make_forecasting_scorer attribute error
**Describe the bug**
<!--
A clear and concise description of what the bug is.
-->
When trying to create a custom forecasting scorer with `make_forecasting_scorer`, it fails with `AttributeError: '_DynamicForecastingErrorMetric' object has no attribute 'lower_is_better'`.

**To Reproduce**
<!--
Add a Minimal, Complete, and Verifiable example (for more details, see e.g. https://stackoverflow.com/help/mcve

If the code is too long, feel free to put it in a public gist and link it in the issue: https://gist.github.com
-->

```python
from sktime.performance_metrics.forecasting import make_forecasting_scorer
import numpy as np
from sktime.datasets import load_airline

y = load_airline()

def custom_mape(y_true , y_pred) -> float:

    eps = np.finfo(np.float64).eps

    result = np.mean(np.abs(y_true - y_pred) / np.maximum(np.abs(y_true), eps))

    return float(result)


fc_scorer = make_forecasting_scorer(
    func=custom_mape,
    name="custom_mape",
    greater_is_better=False,
)

fc_scorer(y,y)
```

**Expected behavior**
<!--
A clear and concise description of what you expected to happen.
-->
Creating custom_score should be done without any error.

**Additional context**
<!--
Add any other context about the problem here.
-->
This was working as expected in earlier version IIRC. I think this is happens after `metric refactor` implementations.

**Versions**
<details>

<!--
Please run the following code snippet and paste the output here:
 
from sktime import show_versions; show_versions()
-->
System:
    python: 3.7.13 (default, Apr 24 2022, 01:04:09)  [GCC 7.5.0]
executable: /usr/bin/python3
   machine: Linux-5.4.188+-x86_64-with-Ubuntu-18.04-bionic

Python dependencies:
          pip: 21.1.3
   setuptools: 57.4.0
      sklearn: 1.0.2
       sktime: 0.13.0
  statsmodels: 0.13.2
        numpy: 1.21.6
        scipy: 1.7.3
       pandas: 1.3.5
   matplotlib: 3.2.2
       joblib: 1.1.0
        numba: 0.56.0
     pmdarima: 1.8.5
      tsfresh: None
</details>

<!-- Thanks for contributing! -->



Discussion:

Resolving patch:
diff --git a/sktime/performance_metrics/forecasting/_classes.py b/sktime/performance_metrics/forecasting/_classes.py
--- a/sktime/performance_metrics/forecasting/_classes.py
+++ b/sktime/performance_metrics/forecasting/_classes.py
@@ -8,6 +8,7 @@
 the lower the better.
 """
 from copy import deepcopy
+from inspect import getfullargspec, isfunction, signature
 from warnings import warn
 
 import numpy as np
@@ -121,7 +122,11 @@ class BaseForecastingErrorMetric(BaseMetric):
     def __init__(self, multioutput="uniform_average", multilevel="uniform_average"):
         self.multioutput = multioutput
         self.multilevel = multilevel
-        self.name = type(self).__name__
+
+        if not hasattr(self, "name"):
+            self.name = type(self).__name__
+
+        super(BaseForecastingErrorMetric, self).__init__()
 
     def __call__(self, y_true, y_pred, **kwargs):
         """Calculate metric value using underlying metric function.
@@ -509,10 +514,24 @@ def _evaluate(self, y_true, y_pred, **kwargs):
         """Evaluate the desired metric on given inputs."""
         # this dict should contain all parameters
         params = self.get_params()
+
         # adding kwargs to the metric, should not overwrite params (but does if clashes)
         params.update(kwargs)
+
+        # calls class variable func, if available, or dynamic (object) variable
         # we need to call type since we store func as a class attribute
-        res = type(self).func(y_true=y_true, y_pred=y_pred, **params)
+        if hasattr(type(self), "func") and isfunction(type(self).func):
+            func = type(self).func
+        else:
+            func = self.func
+
+        # if func does not catch kwargs, subset to args of func
+        if getfullargspec(func).varkw is None:
+            func_params = signature(func).parameters.keys()
+            func_params = set(func_params).difference(["y_true", "y_pred"])
+            params = {key: params[key] for key in func_params}
+
+        res = func(y_true=y_true, y_pred=y_pred, **params)
         return res
 
 
@@ -531,10 +550,43 @@ def __init__(
         self.multilevel = multilevel
         self.func = func
         self.name = name
-        super().__init__()
+        self.lower_is_better = lower_is_better
+
+        super(_DynamicForecastingErrorMetric, self).__init__(
+            multioutput=multioutput, multilevel=multilevel
+        )
+
+        self.set_tags(**{"lower_is_better": lower_is_better})
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the estimator.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return `"default"` set.
+
+        Returns
+        -------
+        params : dict or list of dict, default = {}
+            Parameters to create testing instances of the class
+            Each dict are parameters to construct an "interesting" test instance, i.e.,
+            `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance.
+            `create_test_instance` uses the first (or only) dictionary in `params`
+        """
+
+        def custom_mape(y_true, y_pred) -> float:
+
+            eps = np.finfo(np.float64).eps
+
+            result = np.mean(np.abs(y_true - y_pred) / np.maximum(np.abs(y_true), eps))
+
+            return float(result)
 
-        if not lower_is_better:
-            self.set_tags(**{"lower_is_better": False})
+        params = {"func": custom_mape, "name": "custom_mape", "lower_is_better": False}
+        return params
 
 
 class _ScaledMetricTags:
diff --git a/sktime/performance_metrics/forecasting/_functions.py b/sktime/performance_metrics/forecasting/_functions.py
--- a/sktime/performance_metrics/forecasting/_functions.py
+++ b/sktime/performance_metrics/forecasting/_functions.py
@@ -54,7 +54,7 @@ def _get_kwarg(kwarg, metric_name="Metric", **kwargs):
     if kwarg_ is None:
         msg = "".join(
             [
-                f"{metric_name} requires `{kwarg}`.",
+                f"{metric_name} requires `{kwarg}`. ",
                 f"Pass `{kwarg}` as a keyword argument when calling the metric.",
             ]
         )
diff --git a/sktime/utils/estimator_checks.py b/sktime/utils/estimator_checks.py
--- a/sktime/utils/estimator_checks.py
+++ b/sktime/utils/estimator_checks.py
@@ -4,6 +4,8 @@
 __author__ = ["fkiraly"]
 __all__ = ["check_estimator"]
 
+from inspect import isclass
+
 
 def check_estimator(
     estimator,
@@ -96,7 +98,14 @@ def check_estimator(
         fixtures_to_exclude=fixtures_to_exclude,
     )
 
-    if isinstance(estimator, BaseEstimator) or issubclass(estimator, BaseEstimator):
+    def is_estimator(obj):
+        """Return whether obj is an estimator class or estimator object."""
+        if isclass(obj):
+            return issubclass(obj, BaseEstimator)
+        else:
+            return isinstance(obj, BaseEstimator)
+
+    if is_estimator(estimator):
         results_estimator = TestAllEstimators().run_tests(
             estimator=estimator,
             return_exceptions=return_exceptions,


Test patch:
diff --git a/sktime/performance_metrics/tests/test_metrics_classes.py b/sktime/performance_metrics/tests/test_metrics_classes.py
--- a/sktime/performance_metrics/tests/test_metrics_classes.py
+++ b/sktime/performance_metrics/tests/test_metrics_classes.py
@@ -6,7 +6,12 @@
 import pandas as pd
 import pytest
 
-from sktime.performance_metrics.forecasting import MeanSquaredError, _classes
+from sktime.datasets import load_airline
+from sktime.performance_metrics.forecasting import (
+    MeanSquaredError,
+    _classes,
+    make_forecasting_scorer,
+)
 from sktime.utils._testing.hierarchical import _make_hierarchical
 from sktime.utils._testing.series import _make_series
 
@@ -96,3 +101,32 @@ def test_metric_hierarchical(multioutput, multilevel, n_columns):
             assert isinstance(res, np.ndarray)
             assert res.ndim == 1
             assert len(res) == len(y_true.columns)
+
+
+@pytest.mark.parametrize("greater_is_better", [True, False])
+def test_custom_metric(greater_is_better):
+    """Test custom metric constructor, integration _DynamicForecastingErrorMetric."""
+    from sktime.utils.estimator_checks import check_estimator
+
+    y = load_airline()
+
+    def custom_mape(y_true, y_pred) -> float:
+
+        eps = np.finfo(np.float64).eps
+
+        result = np.mean(np.abs(y_true - y_pred) / np.maximum(np.abs(y_true), eps))
+
+        return float(result)
+
+    fc_scorer = make_forecasting_scorer(
+        func=custom_mape,
+        name="custom_mape",
+        greater_is_better=False,
+    )
+
+    assert isinstance(fc_scorer, _classes._DynamicForecastingErrorMetric)
+
+    score = fc_scorer(y, y)
+    assert isinstance(score, float)
+
+    check_estimator(fc_scorer, return_exceptions=False)