Repository: statsmodels/statsmodels

Issue:
Add Root Mean Square Percentage Error as rmspe()
- [ x] closes #6773
- [ ] tests added / passed. 
- [ x] code/documentation is well formatted.  
- [ ]x properly formatted commit message. See 
      [NumPy's guide](https://docs.scipy.org/doc/numpy-1.15.1/dev/gitwash/development_workflow.html#writing-the-commit-message). 




Discussion:

Resolving patch:
diff --git a/statsmodels/tools/eval_measures.py b/statsmodels/tools/eval_measures.py
--- a/statsmodels/tools/eval_measures.py
+++ b/statsmodels/tools/eval_measures.py
@@ -2,8 +2,9 @@
 """some measures for evaluation of prediction, tests and model selection
 
 Created on Tue Nov 08 15:23:20 2011
+Updated on Wed Jun 03 10:42:20 2020
 
-Author: Josef Perktold
+Authors: Josef Perktold & Peter Prescott
 License: BSD-3
 
 """
@@ -37,7 +38,7 @@ def mse(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.mean((x1-x2)**2, axis=axis)
+    return np.mean((x1 - x2) ** 2, axis=axis)
 
 
 def rmse(x1, x2, axis=0):
@@ -68,6 +69,37 @@ def rmse(x1, x2, axis=0):
     return np.sqrt(mse(x1, x2, axis=axis))
 
 
+def rmspe(y, y_hat, axis=0, zeros=np.nan):
+    """
+    Root Mean Squared Percentage Error
+
+    Parameters
+    ----------
+    y : array_like
+      The actual value.
+    y_hat : array_like
+       The predicted value.
+    axis : int
+       Axis along which the summary statistic is calculated
+    zeros : float
+       Value to assign to error where y is zero
+
+    Returns
+    -------
+    rmspe : ndarray or float
+       Root Mean Squared Percentage Error along given axis.
+    """
+    y_hat = np.asarray(y_hat)
+    y = np.asarray(y)
+    error = y - y_hat
+    loc = y != 0
+    loc = loc.ravel()
+    percentage_error = np.full_like(error, zeros)
+    percentage_error.flat[loc] = error.flat[loc] / y.flat[loc]
+    mspe = np.nanmean(percentage_error ** 2, axis=axis) * 100
+    return np.sqrt(mspe)
+
+
 def maxabs(x1, x2, axis=0):
     """maximum absolute error
 
@@ -92,7 +124,7 @@ def maxabs(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.max(np.abs(x1-x2), axis=axis)
+    return np.max(np.abs(x1 - x2), axis=axis)
 
 
 def meanabs(x1, x2, axis=0):
@@ -119,7 +151,7 @@ def meanabs(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.mean(np.abs(x1-x2), axis=axis)
+    return np.mean(np.abs(x1 - x2), axis=axis)
 
 
 def medianabs(x1, x2, axis=0):
@@ -146,7 +178,7 @@ def medianabs(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.median(np.abs(x1-x2), axis=axis)
+    return np.median(np.abs(x1 - x2), axis=axis)
 
 
 def bias(x1, x2, axis=0):
@@ -173,7 +205,7 @@ def bias(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.mean(x1-x2, axis=axis)
+    return np.mean(x1 - x2, axis=axis)
 
 
 def medianbias(x1, x2, axis=0):
@@ -200,7 +232,7 @@ def medianbias(x1, x2, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.median(x1-x2, axis=axis)
+    return np.median(x1 - x2, axis=axis)
 
 
 def vare(x1, x2, ddof=0, axis=0):
@@ -227,7 +259,7 @@ def vare(x1, x2, ddof=0, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.var(x1-x2, ddof=ddof, axis=axis)
+    return np.var(x1 - x2, ddof=ddof, axis=axis)
 
 
 def stde(x1, x2, ddof=0, axis=0):
@@ -254,7 +286,7 @@ def stde(x1, x2, ddof=0, axis=0):
     """
     x1 = np.asanyarray(x1)
     x2 = np.asanyarray(x2)
-    return np.std(x1-x2, ddof=ddof, axis=axis)
+    return np.std(x1 - x2, ddof=ddof, axis=axis)
 
 
 def iqr(x1, x2, axis=0):
@@ -279,15 +311,15 @@ def iqr(x1, x2, axis=0):
     -----
     If ``x1`` and ``x2`` have different shapes, then they must broadcast.
     """
-    x1 = array_like(x1, 'x1', dtype=None, ndim=None)
-    x2 = array_like(x2, 'x1', dtype=None, ndim=None)
+    x1 = array_like(x1, "x1", dtype=None, ndim=None)
+    x2 = array_like(x2, "x1", dtype=None, ndim=None)
     if axis is None:
         x1 = x1.ravel()
         x2 = x2.ravel()
         axis = 0
     xdiff = np.sort(x1 - x2, axis=axis)
     nobs = x1.shape[axis]
-    idx = np.round((nobs-1) * np.array([0.25, 0.75])).astype(int)
+    idx = np.round((nobs - 1) * np.array([0.25, 0.75])).astype(int)
     sl = [slice(None)] * xdiff.ndim
     sl[axis] = idx
     iqr = np.diff(xdiff[tuple(sl)], axis=axis)
@@ -298,6 +330,7 @@ def iqr(x1, x2, axis=0):
 # Information Criteria
 # ---------------------
 
+
 def aic(llf, nobs, df_modelwc):
     """
     Akaike information criterion
@@ -320,7 +353,7 @@ def aic(llf, nobs, df_modelwc):
     ----------
     https://en.wikipedia.org/wiki/Akaike_information_criterion
     """
-    return -2. * llf + 2. * df_modelwc
+    return -2.0 * llf + 2.0 * df_modelwc
 
 
 def aicc(llf, nobs, df_modelwc):
@@ -345,7 +378,7 @@ def aicc(llf, nobs, df_modelwc):
     ----------
     https://en.wikipedia.org/wiki/Akaike_information_criterion#AICc
     """
-    return -2. * llf + 2. * df_modelwc * nobs / (nobs - df_modelwc - 1.)
+    return -2.0 * llf + 2.0 * df_modelwc * nobs / (nobs - df_modelwc - 1.0)
 
 
 def bic(llf, nobs, df_modelwc):
@@ -370,7 +403,7 @@ def bic(llf, nobs, df_modelwc):
     ----------
     https://en.wikipedia.org/wiki/Bayesian_information_criterion
     """
-    return -2. * llf + np.log(nobs) * df_modelwc
+    return -2.0 * llf + np.log(nobs) * df_modelwc
 
 
 def hqic(llf, nobs, df_modelwc):
@@ -395,11 +428,12 @@ def hqic(llf, nobs, df_modelwc):
     ----------
     Wikipedia does not say much
     """
-    return -2. * llf + 2 * np.log(np.log(nobs)) * df_modelwc
+    return -2.0 * llf + 2 * np.log(np.log(nobs)) * df_modelwc
 
 
 # IC based on residual sigma
 
+
 def aic_sigma(sigma2, nobs, df_modelwc, islog=False):
     r"""
     Akaike information criterion
@@ -564,6 +598,24 @@ def hqic_sigma(sigma2, nobs, df_modelwc, islog=False):
 #     ((nobs + self.df_model) / self.df_resid) ** neqs * np.exp(ld)
 
 
-__all__ = [maxabs, meanabs, medianabs, medianbias, mse, rmse, stde, vare,
-           aic, aic_sigma, aicc, aicc_sigma, bias, bic, bic_sigma,
-           hqic, hqic_sigma, iqr]
+__all__ = [
+    maxabs,
+    meanabs,
+    medianabs,
+    medianbias,
+    mse,
+    rmse,
+    rmspe,
+    stde,
+    vare,
+    aic,
+    aic_sigma,
+    aicc,
+    aicc_sigma,
+    bias,
+    bic,
+    bic_sigma,
+    hqic,
+    hqic_sigma,
+    iqr,
+]


Test patch:
diff --git a/statsmodels/tools/tests/test_eval_measures.py b/statsmodels/tools/tests/test_eval_measures.py
--- a/statsmodels/tools/tests/test_eval_measures.py
+++ b/statsmodels/tools/tests/test_eval_measures.py
@@ -4,89 +4,103 @@
 
 @author: josef
 """
-import pytest
-
 import numpy as np
-from numpy.testing import assert_equal, assert_almost_equal
+from numpy.testing import assert_almost_equal, assert_equal
+import pytest
 
 from statsmodels.tools.eval_measures import (
-    maxabs, meanabs, medianabs, medianbias, mse, rmse, vare,
-    aic, aic_sigma, aicc, aicc_sigma, bias, bic, bic_sigma,
-    hqic, hqic_sigma, iqr)
+    aic,
+    aic_sigma,
+    aicc,
+    aicc_sigma,
+    bias,
+    bic,
+    bic_sigma,
+    hqic,
+    hqic_sigma,
+    iqr,
+    maxabs,
+    meanabs,
+    medianabs,
+    medianbias,
+    mse,
+    rmse,
+    rmspe,
+    vare,
+)
 
 
 def test_eval_measures():
-    #mainly regression tests
+    # mainly regression tests
+    x = np.arange(20).reshape(4, 5)
+    y = np.ones((4, 5))
 
-    x = np.arange(20).reshape(4,5)
-    y = np.ones((4,5))
-    assert_equal(iqr(x, y), 5*np.ones(5))
-    assert_equal(iqr(x, y, axis=1), 2*np.ones(4))
+    assert_equal(iqr(x, y), 5 * np.ones(5))
+    assert_equal(iqr(x, y, axis=1), 2 * np.ones(4))
     assert_equal(iqr(x, y, axis=None), 9)
 
-    assert_equal(mse(x, y),
-                 np.array([  73.5,   87.5,  103.5,  121.5,  141.5]))
-    assert_equal(mse(x, y, axis=1),
-                 np.array([   3.,   38.,  123.,  258.]))
-
-    assert_almost_equal(rmse(x, y),
-                        np.array([  8.5732141 ,   9.35414347,  10.17349497,
-                                   11.02270384,  11.89537725]))
-    assert_almost_equal(rmse(x, y, axis=1),
-                        np.array([  1.73205081,   6.164414,
-                                   11.09053651,  16.0623784 ]))
-
-    assert_equal(maxabs(x, y),
-                 np.array([ 14.,  15.,  16.,  17.,  18.]))
-    assert_equal(maxabs(x, y, axis=1),
-                 np.array([  3.,   8.,  13.,  18.]))
-
-    assert_equal(meanabs(x, y),
-                 np.array([  7. ,   7.5,   8.5,   9.5,  10.5]))
-    assert_equal(meanabs(x, y, axis=1),
-                 np.array([  1.4,   6. ,  11. ,  16. ]))
-    assert_equal(meanabs(x, y, axis=0),
-                 np.array([  7. ,   7.5,   8.5,   9.5,  10.5]))
-
-    assert_equal(medianabs(x, y),
-                 np.array([  6.5,   7.5,   8.5,   9.5,  10.5]))
-    assert_equal(medianabs(x, y, axis=1),
-                 np.array([  1.,   6.,  11.,  16.]))
-
-    assert_equal(bias(x, y),
-                 np.array([  6.5,   7.5,   8.5,   9.5,  10.5]))
-    assert_equal(bias(x, y, axis=1),
-                 np.array([  1.,   6.,  11.,  16.]))
-
-    assert_equal(medianbias(x, y),
-                 np.array([  6.5,   7.5,   8.5,   9.5,  10.5]))
-    assert_equal(medianbias(x, y, axis=1),
-                 np.array([  1.,   6.,  11.,  16.]))
-
-    assert_equal(vare(x, y),
-                 np.array([ 31.25,  31.25,  31.25,  31.25,  31.25]))
-    assert_equal(vare(x, y, axis=1),
-                 np.array([ 2.,  2.,  2.,  2.]))
+    assert_equal(mse(x, y), np.array([73.5, 87.5, 103.5, 121.5, 141.5]))
+    assert_equal(mse(x, y, axis=1), np.array([3.0, 38.0, 123.0, 258.0]))
+
+    assert_almost_equal(
+        rmse(x, y),
+        np.array(
+            [8.5732141, 9.35414347, 10.17349497, 11.02270384, 11.89537725]
+        ),
+    )
+    assert_almost_equal(
+        rmse(x, y, axis=1),
+        np.array([1.73205081, 6.164414, 11.09053651, 16.0623784]),
+    )
+
+    err = x - y
+    loc = np.where(x != 0)
+    err[loc] /= x[loc]
+    err[np.where(x == 0)] = np.nan
+    expected = np.sqrt(np.nanmean(err ** 2, 0) * 100)
+    assert_almost_equal(rmspe(x, y), expected)
+    err[np.where(np.isnan(err))] = 0.0
+    expected = np.sqrt(np.nanmean(err ** 2, 0) * 100)
+    assert_almost_equal(rmspe(x, y, zeros=0), expected)
+
+    assert_equal(maxabs(x, y), np.array([14.0, 15.0, 16.0, 17.0, 18.0]))
+    assert_equal(maxabs(x, y, axis=1), np.array([3.0, 8.0, 13.0, 18.0]))
+
+    assert_equal(meanabs(x, y), np.array([7.0, 7.5, 8.5, 9.5, 10.5]))
+    assert_equal(meanabs(x, y, axis=1), np.array([1.4, 6.0, 11.0, 16.0]))
+    assert_equal(meanabs(x, y, axis=0), np.array([7.0, 7.5, 8.5, 9.5, 10.5]))
+
+    assert_equal(medianabs(x, y), np.array([6.5, 7.5, 8.5, 9.5, 10.5]))
+    assert_equal(medianabs(x, y, axis=1), np.array([1.0, 6.0, 11.0, 16.0]))
+
+    assert_equal(bias(x, y), np.array([6.5, 7.5, 8.5, 9.5, 10.5]))
+    assert_equal(bias(x, y, axis=1), np.array([1.0, 6.0, 11.0, 16.0]))
+
+    assert_equal(medianbias(x, y), np.array([6.5, 7.5, 8.5, 9.5, 10.5]))
+    assert_equal(medianbias(x, y, axis=1), np.array([1.0, 6.0, 11.0, 16.0]))
+
+    assert_equal(vare(x, y), np.array([31.25, 31.25, 31.25, 31.25, 31.25]))
+    assert_equal(vare(x, y, axis=1), np.array([2.0, 2.0, 2.0, 2.0]))
 
 
 ics = [aic, aicc, bic, hqic]
 ics_sig = [aic_sigma, aicc_sigma, bic_sigma, hqic_sigma]
 
 
-@pytest.mark.parametrize('ic,ic_sig', zip(ics, ics_sig))
+@pytest.mark.parametrize("ic,ic_sig", zip(ics, ics_sig))
 def test_ic_equivalence(ic, ic_sig):
     # consistency check
 
     assert ic(np.array(2), 10, 2).dtype == float
     assert ic_sig(np.array(2), 10, 2).dtype == float
 
-    assert_almost_equal(ic(-10./2.*np.log(2.), 10, 2)/10,
-                        ic_sig(2, 10, 2),
-                        decimal=14)
+    assert_almost_equal(
+        ic(-10.0 / 2.0 * np.log(2.0), 10, 2) / 10, ic_sig(2, 10, 2), decimal=14
+    )
 
-    assert_almost_equal(ic_sig(np.log(2.), 10, 2, islog=True),
-                        ic_sig(2, 10, 2),
-                        decimal=14)
+    assert_almost_equal(
+        ic_sig(np.log(2.0), 10, 2, islog=True), ic_sig(2, 10, 2), decimal=14
+    )
 
 
 def test_ic():
@@ -95,12 +109,15 @@ def test_ic():
     # examples penalty directly from formula
     n = 10
     k = 2
-    assert_almost_equal(aic(0, 10, 2), 2*k, decimal=14)
+    assert_almost_equal(aic(0, 10, 2), 2 * k, decimal=14)
     # next see Wikipedia
-    assert_almost_equal(aicc(0, 10, 2),
-                        aic(0, n, k) + 2*k*(k+1.)/(n-k-1.), decimal=14)
-    assert_almost_equal(bic(0, 10, 2), np.log(n)*k, decimal=14)
-    assert_almost_equal(hqic(0, 10, 2), 2*np.log(np.log(n))*k, decimal=14)
+    assert_almost_equal(
+        aicc(0, 10, 2),
+        aic(0, n, k) + 2 * k * (k + 1.0) / (n - k - 1.0),
+        decimal=14,
+    )
+    assert_almost_equal(bic(0, 10, 2), np.log(n) * k, decimal=14)
+    assert_almost_equal(hqic(0, 10, 2), 2 * np.log(np.log(n)) * k, decimal=14)
 
 
 def test_iqr_axis(reset_randomstate):
@@ -112,12 +129,12 @@ def test_iqr_axis(reset_randomstate):
 
     ax_0 = iqr(x1, x2, axis=0)
     assert ax_0.shape == (100,)
-    ax_0_direct = [iqr(x1[:,i], x2[:,i]) for i in range(100)]
+    ax_0_direct = [iqr(x1[:, i], x2[:, i]) for i in range(100)]
     assert_almost_equal(ax_0, np.array(ax_0_direct))
 
     ax_1 = iqr(x1, x2, axis=1)
     assert ax_1.shape == (100,)
-    ax_1_direct = [iqr(x1[i,:], x2[i,:]) for i in range(100)]
+    ax_1_direct = [iqr(x1[i, :], x2[i, :]) for i in range(100)]
     assert_almost_equal(ax_1, np.array(ax_1_direct))
 
     assert any(ax_0 != ax_1)