Repository: awslabs/gluonts

Issue:
MAPE metric calculation in GLUONTS
## Description
The calculation of the MAPE metric looks strange. You are removing zero actual values but still you calculate the mean including the zeros. Any explanation for that?

```python
    @staticmethod
    def mape(target, forecast):
        r"""
        .. math::

            mape = mean(|Y - Y_hat| / |Y|))
        """

        denominator = np.abs(target)
        flag = denominator == 0

        mape = np.mean(
            (np.abs(target - forecast) * (1 - flag)) / (denominator + flag)
        )
        return mape
```


Discussion:
Hello @ammarwm,

thanks for raising the issue :)

#1135 seems related.

I've talked with @lostella, and we want to generally improve and fix how metrics are calculated. 


Resolving patch:
diff --git a/src/gluonts/evaluation/_base.py b/src/gluonts/evaluation/_base.py
--- a/src/gluonts/evaluation/_base.py
+++ b/src/gluonts/evaluation/_base.py
@@ -35,6 +35,10 @@
 from gluonts.time_feature import get_seasonality
 
 
+def nan_if_masked(a: Union[float, np.ma.core.MaskedConstant]) -> float:
+    return a if a is not np.ma.masked else np.nan
+
+
 class Evaluator:
     """
     Evaluator class, to compute accuracy metrics by comparing observations
@@ -81,7 +85,6 @@ class Evaluator:
     """
 
     default_quantiles = 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9
-    zero_tol = 1e-8
 
     def __init__(
         self,
@@ -233,9 +236,7 @@ def extract_past_data(
             np.squeeze(time_series.loc[:date_before_forecast].transpose())
         )
 
-    def seasonal_error(
-        self, past_data: np.ndarray, forecast: Forecast
-    ) -> float:
+    def seasonal_error(self, past_data: np.ndarray, forecast: Forecast):
         r"""
         .. math::
 
@@ -262,7 +263,7 @@ def seasonal_error(
 
         seasonal_mae = np.mean(abs(y_t - y_tm))
 
-        return seasonal_mae if seasonal_mae is not np.ma.masked else np.nan
+        return nan_if_masked(seasonal_mae)
 
     def get_metrics_per_ts(
         self, time_series: Union[pd.Series, pd.DataFrame], forecast: Forecast
@@ -280,7 +281,7 @@ def get_metrics_per_ts(
             mean_fcst = None
         median_fcst = forecast.quantile(0.5)
         seasonal_error = self.seasonal_error(past_data, forecast)
-        metrics = {
+        metrics: Dict[str, Union[float, str, None]] = {
             "item_id": forecast.item_id,
             "MSE": self.mse(pred_target, mean_fcst)
             if mean_fcst is not None
@@ -387,36 +388,23 @@ def get_aggregate_metrics(
 
         # derived metrics based on previous aggregate metrics
         totals["RMSE"] = np.sqrt(totals["MSE"])
-
-        flag = totals["abs_target_mean"] <= Evaluator.zero_tol
-        totals["NRMSE"] = np.divide(
-            totals["RMSE"] * (1 - flag), totals["abs_target_mean"] + flag
-        )
-
-        flag = totals["abs_target_sum"] <= Evaluator.zero_tol
-        totals["ND"] = np.divide(
-            totals["abs_error"] * (1 - flag), totals["abs_target_sum"] + flag
-        )
-
-        all_qLoss_names = [
-            quantile.weighted_loss_name for quantile in self.quantiles
-        ]
-
-        all_abs_qLoss_names = [
-            quantile.loss_name for quantile in self.quantiles
-        ]
+        totals["NRMSE"] = totals["RMSE"] / totals["abs_target_mean"]
+        totals["ND"] = totals["abs_error"] / totals["abs_target_sum"]
 
         for quantile in self.quantiles:
-            totals[quantile.weighted_loss_name] = np.divide(
-                totals[quantile.loss_name], totals["abs_target_sum"] + flag
+            totals[quantile.weighted_loss_name] = (
+                totals[quantile.loss_name] / totals["abs_target_sum"]
             )
 
         totals["mean_absolute_QuantileLoss"] = np.array(
-            [totals[abs_ql] for abs_ql in all_abs_qLoss_names]
+            [totals[quantile.loss_name] for quantile in self.quantiles]
         ).mean()
 
         totals["mean_wQuantileLoss"] = np.array(
-            [totals[ql] for ql in all_qLoss_names]
+            [
+                totals[quantile.weighted_loss_name]
+                for quantile in self.quantiles
+            ]
         ).mean()
 
         totals["MAE_Coverage"] = np.mean(
@@ -428,28 +416,33 @@ def get_aggregate_metrics(
         return totals, metric_per_ts
 
     @staticmethod
-    def mse(target, forecast):
-        return np.mean(np.square(target - forecast))
+    def mse(target: np.ndarray, forecast: np.ndarray) -> float:
+        return nan_if_masked(np.mean(np.square(target - forecast)))
 
     @staticmethod
-    def abs_error(target, forecast):
-        return np.sum(np.abs(target - forecast))
+    def abs_error(target: np.ndarray, forecast: np.ndarray) -> float:
+        return nan_if_masked(np.sum(np.abs(target - forecast)))
 
     @staticmethod
-    def quantile_loss(target, quantile_forecast, q):
-        return 2.0 * np.sum(
-            np.abs(
-                (quantile_forecast - target)
-                * ((target <= quantile_forecast) - q)
-            )
+    def quantile_loss(
+        target: np.ndarray, forecast: np.ndarray, q: float
+    ) -> float:
+        return nan_if_masked(
+            2
+            * np.sum(np.abs((forecast - target) * ((target <= forecast) - q)))
         )
 
     @staticmethod
-    def coverage(target, quantile_forecast):
-        return np.mean((target < quantile_forecast))
+    def coverage(target: np.ndarray, forecast: np.ndarray) -> float:
+        return nan_if_masked(np.mean((target < forecast)))
 
     @staticmethod
-    def mase(target, forecast, seasonal_error):
+    def mase(
+        target: np.ndarray,
+        forecast: np.ndarray,
+        seasonal_error: float,
+        exclude_zero_denominator=True,
+    ) -> float:
         r"""
         .. math::
 
@@ -457,44 +450,48 @@ def mase(target, forecast, seasonal_error):
 
         https://www.m4.unic.ac.cy/wp-content/uploads/2018/03/M4-Competitors-Guide.pdf
         """
-        flag = seasonal_error <= Evaluator.zero_tol
-        return (np.mean(np.abs(target - forecast)) * (1 - flag)) / (
-            seasonal_error + flag
+        if exclude_zero_denominator and np.isclose(seasonal_error, 0.0):
+            return np.nan
+
+        return nan_if_masked(
+            np.mean(np.abs(target - forecast)) / seasonal_error
         )
 
     @staticmethod
-    def mape(target, forecast):
+    def mape(
+        target: np.ndarray, forecast: np.ndarray, exclude_zero_denominator=True
+    ) -> float:
         r"""
         .. math::
 
             mape = mean(|Y - Y_hat| / |Y|))
         """
-
         denominator = np.abs(target)
-        flag = denominator <= Evaluator.zero_tol
-
-        mape = np.mean(
-            (np.abs(target - forecast) * (1 - flag)) / (denominator + flag)
-        )
-        return mape
+        if exclude_zero_denominator:
+            denominator = np.ma.masked_where(
+                np.isclose(denominator, 0.0), denominator
+            )
+        return nan_if_masked(np.mean(np.abs(target - forecast) / denominator))
 
     @staticmethod
-    def smape(target, forecast):
+    def smape(
+        target: np.ndarray, forecast: np.ndarray, exclude_zero_denominator=True
+    ) -> float:
         r"""
         .. math::
 
-            smape = mean(2 * |Y - Y_hat| / (|Y| + |Y_hat|))
+            smape = 2 * mean(|Y - Y_hat| / (|Y| + |Y_hat|))
 
         https://www.m4.unic.ac.cy/wp-content/uploads/2018/03/M4-Competitors-Guide.pdf
         """
-
         denominator = np.abs(target) + np.abs(forecast)
-        flag = denominator <= Evaluator.zero_tol
-
-        smape = 2 * np.mean(
-            (np.abs(target - forecast) * (1 - flag)) / (denominator + flag)
+        if exclude_zero_denominator:
+            denominator = np.ma.masked_where(
+                np.isclose(denominator, 0.0), denominator
+            )
+        return nan_if_masked(
+            2 * np.mean(np.abs(target - forecast) / denominator)
         )
-        return smape
 
     @staticmethod
     def owa(
@@ -533,14 +530,24 @@ def owa(
         return owa
 
     @staticmethod
-    def msis(target, lower_quantile, upper_quantile, seasonal_error, alpha):
+    def msis(
+        target: np.ndarray,
+        lower_quantile: np.ndarray,
+        upper_quantile: np.ndarray,
+        seasonal_error: float,
+        alpha: float,
+        exclude_zero_denominator=True,
+    ) -> float:
         r"""
         :math:
 
-            msis = mean(U - L + 2/alpha * (L-Y) * I[Y<L] + 2/alpha * (Y-U) * I[Y>U]) /seasonal_error
+            msis = mean(U - L + 2/alpha * (L-Y) * I[Y<L] + 2/alpha * (Y-U) * I[Y>U]) / seasonal_error
 
         https://www.m4.unic.ac.cy/wp-content/uploads/2018/03/M4-Competitors-Guide.pdf
         """
+        if exclude_zero_denominator and np.isclose(seasonal_error, 0.0):
+            return np.nan
+
         numerator = np.mean(
             upper_quantile
             - lower_quantile
@@ -554,16 +561,15 @@ def msis(target, lower_quantile, upper_quantile, seasonal_error, alpha):
             * (target > upper_quantile)
         )
 
-        flag = seasonal_error <= Evaluator.zero_tol
-        return (numerator * (1 - flag)) / (seasonal_error + flag)
+        return nan_if_masked(numerator / seasonal_error)
 
     @staticmethod
-    def abs_target_sum(target):
-        return np.sum(np.abs(target))
+    def abs_target_sum(target) -> float:
+        return nan_if_masked(np.sum(np.abs(target)))
 
     @staticmethod
-    def abs_target_mean(target):
-        return np.mean(np.abs(target))
+    def abs_target_mean(target) -> float:
+        return nan_if_masked(np.mean(np.abs(target)))
 
 
 class MultivariateEvaluator(Evaluator):


Test patch:
diff --git a/test/evaluation/test_evaluator.py b/test/evaluation/test_evaluator.py
--- a/test/evaluation/test_evaluator.py
+++ b/test/evaluation/test_evaluator.py
@@ -21,6 +21,46 @@
 QUANTILES = [str(q / 10.0) for q in range(1, 10)]
 
 
+@pytest.mark.parametrize(
+    "target, forecast, expected_metric_values",
+    [
+        (
+            np.array([1.0, 2.0, 3.0, 0.0, 0.0]),
+            np.array([2.0, 3.0, 4.0, 1.0, 0.0]),
+            {
+                Evaluator.mape: (1 / 1 + 1 / 2 + 1 / 3) / 3,
+                Evaluator.smape: 2
+                * (1 / (1 + 2) + 1 / (2 + 3) + 1 / (3 + 4) + 1 / 1)
+                / 4,
+            },
+        ),
+        (
+            np.array([0.0, 0.0, 0.0, 0.0, 0.0]),
+            np.array([0.1, 0.01, 0.001, 0.0001, 0.00001]),
+            {
+                Evaluator.mape: np.nan,
+                Evaluator.smape: 2.0,
+            },
+        ),
+        (
+            np.array([0.0, 0.0, 0.0, 0.0, 0.0]),
+            np.array([0.0, 0.0, 0.0, 0.0, 0.0]),
+            {
+                Evaluator.mape: np.nan,
+                Evaluator.smape: np.nan,
+            },
+        ),
+    ],
+)
+def test_metric_values(
+    target: np.ndarray, forecast: np.ndarray, expected_metric_values: dict
+):
+    for metric, expected_value in expected_metric_values.items():
+        assert np.isclose(
+            metric(target, forecast), expected_value, equal_nan=True
+        )
+
+
 def data_iterator(ts):
     """
     :param ts: list of pd.Series or pd.DataFrame
@@ -329,16 +369,16 @@ def test_MASE_sMAPE_M4(timeseries, res):
         "abs_target_sum": 0.0,
         "abs_target_mean": 0.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
-        "MAPE": 0.0,
-        "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MASE": np.nan,
+        "MAPE": np.nan,
+        "sMAPE": np.nan,
+        "MSIS": np.nan,
         "RMSE": 0.0,
-        "NRMSE": 0.0,
-        "ND": 0.0,
+        "NRMSE": np.nan,
+        "ND": np.nan,
         "MAE_Coverage": 0.5,
         "mean_absolute_QuantileLoss": 0.0,
-        "mean_wQuantileLoss": 0.0,
+        "mean_wQuantileLoss": np.nan,
     },
     {
         "MSE": 0.0,
@@ -346,10 +386,10 @@ def test_MASE_sMAPE_M4(timeseries, res):
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -363,10 +403,10 @@ def test_MASE_sMAPE_M4(timeseries, res):
         "abs_target_sum": 14.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -414,10 +454,10 @@ def test_MASE_sMAPE_M4(timeseries, res):
         "abs_target_sum": 3.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -450,7 +490,7 @@ def test_metrics(timeseries, res, has_nans, input_type):
 
     for metric, score in agg_metrics.items():
         if metric in res.keys():
-            assert abs(score - res[metric]) < 0.001, (
+            assert np.isclose(score, res[metric], equal_nan=True), (
                 "Scores for the metric {} do not match: \nexpected: {} "
                 "\nobtained: {}".format(metric, res[metric], score)
             )
@@ -474,7 +514,7 @@ def test_metrics_mp(timeseries, res, has_nans, input_type):
 
     for metric, score in agg_metrics.items():
         if metric in res.keys():
-            assert abs(score - res[metric]) < 0.001, (
+            assert np.isclose(score, res[metric], equal_nan=True), (
                 "Scores for the metric {} do not match: \nexpected: {} "
                 "\nobtained: {}".format(metric, res[metric], score)
             )
@@ -516,9 +556,9 @@ def test_metrics_mp(timeseries, res, has_nans, input_type):
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -531,9 +571,9 @@ def test_metrics_mp(timeseries, res, has_nans, input_type):
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -546,9 +586,9 @@ def test_metrics_mp(timeseries, res, has_nans, input_type):
         "abs_target_sum": 30.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -640,7 +680,7 @@ def test_metrics_multivariate(
 
     for metric, score in agg_metrics.items():
         if metric in res.keys():
-            assert abs(score - res[metric]) < 0.001, (
+            assert np.isclose(score, res[metric], equal_nan=True), (
                 "Scores for the metric {} do not match: \nexpected: {} "
                 "\nobtained: {}".format(metric, res[metric], score)
             )
@@ -698,16 +738,16 @@ def rmsle(target, forecast):
         "abs_target_sum": 0.0,
         "abs_target_mean": 0.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
-        "MAPE": 0.0,
-        "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MASE": np.nan,
+        "MAPE": np.nan,
+        "sMAPE": np.nan,
+        "MSIS": np.nan,
         "RMSE": 0.0,
-        "NRMSE": 0.0,
-        "ND": 0.0,
+        "NRMSE": np.nan,
+        "ND": np.nan,
         "MAE_Coverage": 0.5,
         "mean_absolute_QuantileLoss": 0.0,
-        "mean_wQuantileLoss": 0.0,
+        "mean_wQuantileLoss": np.nan,
     },
     {
         "RMSLE": 0.0,
@@ -716,10 +756,10 @@ def rmsle(target, forecast):
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -734,10 +774,10 @@ def rmsle(target, forecast):
         "abs_target_sum": 14.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -824,10 +864,10 @@ def rmsle(target, forecast):
         "abs_target_sum": 3.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "MAPE": 0.0,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -899,7 +939,7 @@ def test_custom_eval_fn(
 
     for metric, score in agg_metrics.items():
         if metric in res.keys():
-            assert abs(score - res[metric]) < 0.001, (
+            assert np.isclose(score, res[metric], equal_nan=True), (
                 "Scores for the metric {} do not match: \nexpected: {} "
                 "\nobtained: {}".format(metric, res[metric], score)
             )
@@ -942,9 +982,9 @@ def test_custom_eval_fn(
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -958,9 +998,9 @@ def test_custom_eval_fn(
         "abs_target_sum": 15.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -974,9 +1014,9 @@ def test_custom_eval_fn(
         "abs_target_sum": 30.0,
         "abs_target_mean": 1.0,
         "seasonal_error": 0.0,
-        "MASE": 0.0,
+        "MASE": np.nan,
         "sMAPE": 0.0,
-        "MSIS": 0.0,
+        "MSIS": np.nan,
         "RMSE": 0.0,
         "NRMSE": 0.0,
         "ND": 0.0,
@@ -1089,7 +1129,7 @@ def test_metrics_multivariate_custom_eval_fn(
 
     for metric, score in agg_metrics.items():
         if metric in res.keys():
-            assert abs(score - res[metric]) < 0.001, (
+            assert np.isclose(score, res[metric], equal_nan=True), (
                 "Scores for the metric {} do not match: \nexpected: {} "
                 "\nobtained: {}".format(metric, res[metric], score)
             )
diff --git a/test/shell/test_shell.py b/test/shell/test_shell.py
--- a/test/shell/test_shell.py
+++ b/test/shell/test_shell.py
@@ -127,7 +127,7 @@ def test_train_shell(train_env: TrainEnv, caplog, forecaster_type) -> None:
             if "local, Coverage" in line:
                 assert line.endswith("0.0")
             if "MASE" in line or "MSIS" in line:
-                assert line.endswith("0.0")
+                assert line.endswith("nan")
             if "abs_target_sum" in line:
                 assert line.endswith("270.0")