Repository: statsmodels/statsmodels

Issue:
BUG: stats.moment_helpers  does inplace modification
`mnc2cum(m)` modifies the input array inplace

just took me a while to figure out why my numbers don't match up, and keep changing.



Discussion:
It also doesn't work correctly vectorized for 3rd and 4th cumulant

for smmh.mc2cum

```
>>> mcp
array([[   5.,   10.,    1.],
       [   5.,   10.,    1.],
       [   5.,   10.,    1.],
       [  80.,  310.,    4.]])
>>> np.array(smmh.mc2cum(mcp))
array([[  5.00000000e+00,   1.00000000e+01,   1.00000000e+00],
       [  5.00000000e+00,   1.00000000e+01,   1.00000000e+00],
       [  1.30000000e+02,   1.01000000e+03,   2.00000000e+00],
       [ -1.12000000e+03,  -2.39900000e+04,   4.00000000e+00]])

>>> [np.array(smmh.mc2cum(m_)) for m_ in mcp.T]
[array([ 5.,  5.,  5.,  5.]), array([ 10.,  10.,  10.,  10.]), array([ 1.,  1.,  1.,  1.])]
```

I think the vectorized bug is a consequence of inplace modification which is not a problem in the scalar case, because scalars are not mutable.

@josef-pkt  Fix: https://github.com/statsmodels/statsmodels/pull/3362

Resolving patch:
diff --git a/statsmodels/stats/moment_helpers.py b/statsmodels/stats/moment_helpers.py
--- a/statsmodels/stats/moment_helpers.py
+++ b/statsmodels/stats/moment_helpers.py
@@ -1,4 +1,4 @@
-'''helper functions conversion between moments
+"""helper functions conversion between moments
 
 contains:
 
@@ -10,150 +10,214 @@
 Author: Josef Perktold
 License: BSD-3
 
-'''
-
+"""
 from statsmodels.compat.python import range
 
 import numpy as np
 from scipy.special import comb
 
 
-## start moment helpers
+def _convert_to_multidim(x):
+    if any([isinstance(x, list), isinstance(x, tuple)]):
+        return np.array(x)
+    elif isinstance(x, np.ndarray):
+        return x
+    else:
+        # something strange was passed and the function probably
+        # will fall, maybe insert an exception?
+        return x
+
+
+def _convert_from_multidim(x, totype=list):
+    if len(x.shape) < 2:
+        return totype(x)
+    return x.T
+
 
 def mc2mnc(mc):
-    '''convert central to non-central moments, uses recursive formula
+    """convert central to non-central moments, uses recursive formula
     optionally adjusts first moment to return mean
 
-    '''
-    n = len(mc)
-    mean = mc[0]
-    mc = [1] + list(mc)    # add zero moment = 1
-    mc[1] = 0  # define central mean as zero for formula
-    mnc = [1, mean] # zero and first raw moments
-    for nn,m in enumerate(mc[2:]):
-        n=nn+2
-        mnc.append(0)
-        for k in range(n+1):
-            mnc[n] += comb(n,k,exact=1) * mc[k] * mean**(n-k)
+    """
+    x = _convert_to_multidim(mc)
+
+    def _local_counts(mc):
+        mean = mc[0]
+        mc = [1] + list(mc)  # add zero moment = 1
+        mc[1] = 0  # define central mean as zero for formula
+        mnc = [1, mean]  # zero and first raw moments
+        for nn, m in enumerate(mc[2:]):
+            n = nn + 2
+            mnc.append(0)
+            for k in range(n + 1):
+                mnc[n] += comb(n, k, exact=True) * mc[k] * mean ** (n - k)
+        return mnc[1:]
 
-    return mnc[1:]
+    res = np.apply_along_axis(_local_counts, 0, x)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res)
 
 
-def mnc2mc(mnc, wmean = True):
-    '''convert non-central to central moments, uses recursive formula
+def mnc2mc(mnc, wmean=True):
+    """convert non-central to central moments, uses recursive formula
     optionally adjusts first moment to return mean
 
-    '''
-    n = len(mnc)
-    mean = mnc[0]
-    mnc = [1] + list(mnc)    # add zero moment = 1
-    mu = [] #np.zeros(n+1)
-    for n,m in enumerate(mnc):
-        mu.append(0)
-        #[comb(n-1,k,exact=1) for k in range(n)]
-        for k in range(n+1):
-            mu[n] += (-1)**(n-k) * comb(n,k,exact=1) * mnc[k] * mean**(n-k)
-    if wmean:
-        mu[1] = mean
-    return mu[1:]
+    """
+    X = _convert_to_multidim(mnc)
+
+    def _local_counts(mnc):
+        mean = mnc[0]
+        mnc = [1] + list(mnc)  # add zero moment = 1
+        mu = []
+        for n, m in enumerate(mnc):
+            mu.append(0)
+            for k in range(n + 1):
+                sgn_comb = (-1) ** (n - k) * comb(n, k, exact=True)
+                mu[n] += sgn_comb * mnc[k] * mean ** (n - k)
+        if wmean:
+            mu[1] = mean
+        return mu[1:]
+
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res)
 
 
 def cum2mc(kappa):
-    '''convert non-central moments to cumulants
+    """convert non-central moments to cumulants
     recursive formula produces as many cumulants as moments
 
     References
     ----------
     Kenneth Lange: Numerical Analysis for Statisticians, page 40
-    (http://books.google.ca/books?id=gm7kwttyRT0C&pg=PA40&lpg=PA40&dq=convert+cumulants+to+moments&source=web&ots=qyIaY6oaWH&sig=cShTDWl-YrWAzV7NlcMTRQV6y0A&hl=en&sa=X&oi=book_result&resnum=1&ct=result)
-
-
-    '''
-    mc = [1,0.0] #_kappa[0]]  #insert 0-moment and mean
-    kappa0 = kappa[0]
-    kappa = [1] + list(kappa)
-    for nn,m in enumerate(kappa[2:]):
-        n = nn+2
-        mc.append(0)
-        for k in range(n-1):
-            mc[n] += comb(n-1,k,exact=1) * kappa[n-k]*mc[k]
-
-    mc[1] = kappa0 # insert mean as first moments by convention
-    return mc[1:]
+    """
+    X = _convert_to_multidim(kappa)
+
+    def _local_counts(kappa):
+        mc = [1, 0.0]  # _kappa[0]]  #insert 0-moment and mean
+        kappa0 = kappa[0]
+        kappa = [1] + list(kappa)
+        for nn, m in enumerate(kappa[2:]):
+            n = nn + 2
+            mc.append(0)
+            for k in range(n - 1):
+                mc[n] += comb(n - 1, k, exact=True) * kappa[n - k] * mc[k]
+        mc[1] = kappa0  # insert mean as first moments by convention
+        return mc[1:]
+
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res)
 
 
 def mnc2cum(mnc):
-    '''convert non-central moments to cumulants
+    """convert non-central moments to cumulants
     recursive formula produces as many cumulants as moments
 
     http://en.wikipedia.org/wiki/Cumulant#Cumulants_and_moments
-    '''
-    mnc = [1] + list(mnc)
-    kappa = [1]
-    for nn,m in enumerate(mnc[1:]):
-        n = nn+1
-        kappa.append(m)
-        for k in range(1,n):
-            kappa[n] -= comb(n-1,k-1,exact=1) * kappa[k]*mnc[n-k]
+    """
+    X = _convert_to_multidim(mnc)
+
+    def _local_counts(mnc):
+        mnc = [1] + list(mnc)
+        kappa = [1]
+        for nn, m in enumerate(mnc[1:]):
+            n = nn + 1
+            kappa.append(m)
+            for k in range(1, n):
+                num_ways = comb(n - 1, k - 1, exact=True)
+                kappa[n] -= num_ways * kappa[k] * mnc[n - k]
+        return kappa[1:]
 
-    return kappa[1:]
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res)
 
 
 def mc2cum(mc):
-    '''just chained because I have still the test case
-    '''
-    return mnc2cum(mc2mnc(mc))
+    """
+    just chained because I have still the test case
+    """
+    first_step = mc2mnc(mc)
+    if isinstance(first_step, np.ndarray):
+        first_step = first_step.T
+    return mnc2cum(first_step)
+    # return np.apply_along_axis(lambda x: mnc2cum(mc2mnc(x)), 0, mc)
 
 
 def mvsk2mc(args):
-    '''convert mean, variance, skew, kurtosis to central moments'''
-    mu,sig2,sk,kur = args
+    """convert mean, variance, skew, kurtosis to central moments"""
+    X = _convert_to_multidim(args)
 
-    cnt = [None]*4
-    cnt[0] = mu
-    cnt[1] = sig2
-    cnt[2] = sk * sig2**1.5
-    cnt[3] = (kur+3.0) * sig2**2.0
-    return tuple(cnt)
+    def _local_counts(args):
+        mu, sig2, sk, kur = args
+        cnt = [None] * 4
+        cnt[0] = mu
+        cnt[1] = sig2
+        cnt[2] = sk * sig2 ** 1.5
+        cnt[3] = (kur + 3.0) * sig2 ** 2.0
+        return tuple(cnt)
+
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res, tuple)
 
 
 def mvsk2mnc(args):
-    '''convert mean, variance, skew, kurtosis to non-central moments'''
-    mc, mc2, skew, kurt = args
-    mnc = mc
-    mnc2 = mc2 + mc*mc
-    mc3  = skew*(mc2**1.5) # 3rd central moment
-    mnc3 = mc3+3*mc*mc2+mc**3 # 3rd non-central moment
-    mc4  = (kurt+3.0)*(mc2**2.0) # 4th central moment
-    mnc4 = mc4+4*mc*mc3+6*mc*mc*mc2+mc**4
-    return (mnc, mnc2, mnc3, mnc4)
+    """convert mean, variance, skew, kurtosis to non-central moments"""
+    X = _convert_to_multidim(args)
 
+    def _local_counts(args):
+        mc, mc2, skew, kurt = args
+        mnc = mc
+        mnc2 = mc2 + mc * mc
+        mc3 = skew * (mc2 ** 1.5)  # 3rd central moment
+        mnc3 = mc3 + 3 * mc * mc2 + mc ** 3  # 3rd non-central moment
+        mc4 = (kurt + 3.0) * (mc2 ** 2.0)  # 4th central moment
+        mnc4 = mc4 + 4 * mc * mc3 + 6 * mc * mc * mc2 + mc ** 4
+        return (mnc, mnc2, mnc3, mnc4)
 
-def mc2mvsk(args):
-    '''convert central moments to mean, variance, skew, kurtosis
-    '''
-    mc, mc2, mc3, mc4 = args
-    skew = np.divide(mc3, mc2**1.5)
-    kurt = np.divide(mc4, mc2**2.0) - 3.0
-    return (mc, mc2, skew, kurt)
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res, tuple)
 
 
-def mnc2mvsk(args):
-    '''convert central moments to mean, variance, skew, kurtosis
-    '''
-    #convert four non-central moments to central moments
-    mnc, mnc2, mnc3, mnc4 = args
-    mc = mnc
-    mc2 = mnc2 - mnc*mnc
-    mc3 = mnc3 - (3*mc*mc2+mc**3) # 3rd central moment
-    mc4 = mnc4 - (4*mc*mc3+6*mc*mc*mc2+mc**4)
+def mc2mvsk(args):
+    """convert central moments to mean, variance, skew, kurtosis"""
+    X = _convert_to_multidim(args)
+
+    def _local_counts(args):
+        mc, mc2, mc3, mc4 = args
+        skew = np.divide(mc3, mc2 ** 1.5)
+        kurt = np.divide(mc4, mc2 ** 2.0) - 3.0
+        return (mc, mc2, skew, kurt)
 
-    return mc2mvsk((mc, mc2, mc3, mc4))
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res, tuple)
 
 
-#def mnc2mc(args):
-#    '''convert four non-central moments to central moments
-#    '''
+def mnc2mvsk(args):
+    """convert central moments to mean, variance, skew, kurtosis
+    """
+    X = _convert_to_multidim(args)
+
+    def _local_counts(args):
+        # convert four non-central moments to central moments
+        mnc, mnc2, mnc3, mnc4 = args
+        mc = mnc
+        mc2 = mnc2 - mnc * mnc
+        mc3 = mnc3 - (3 * mc * mc2 + mc ** 3)  # 3rd central moment
+        mc4 = mnc4 - (4 * mc * mc3 + 6 * mc * mc * mc2 + mc ** 4)
+        return mc2mvsk((mc, mc2, mc3, mc4))
+
+    res = np.apply_along_axis(_local_counts, 0, X)
+    # for backward compatibility convert 1-dim output to list/tuple
+    return _convert_from_multidim(res, tuple)
+
+# def mnc2mc(args):
+#    """convert four non-central moments to central moments
+#    """
 #    mnc, mnc2, mnc3, mnc4 = args
 #    mc = mnc
 #    mc2 = mnc2 - mnc*mnc
@@ -161,11 +225,12 @@ def mnc2mvsk(args):
 #    mc4 = mnc4 - (4*mc*mc3+6*mc*mc*mc2+mc**4)
 #    return mc, mc2, mc
 
-    #TODO: no return, did it get lost in cut-paste?
+# TODO: no return, did it get lost in cut-paste?
 
 
 def cov2corr(cov, return_std=False):
-    '''convert covariance matrix to correlation matrix
+    """
+    convert covariance matrix to correlation matrix
 
     Parameters
     ----------
@@ -182,10 +247,9 @@ def cov2corr(cov, return_std=False):
 
     Notes
     -----
-    This function does not convert subclasses of ndarrays. This requires
-    that division is defined elementwise. np.ma.array and np.matrix are allowed.
-
-    '''
+    This function does not convert subclasses of ndarrays. This requires that
+    division is defined elementwise. np.ma.array and np.matrix are allowed.
+    """
     cov = np.asanyarray(cov)
     std_ = np.sqrt(np.diag(cov))
     corr = cov / np.outer(std_, std_)
@@ -194,8 +258,10 @@ def cov2corr(cov, return_std=False):
     else:
         return corr
 
+
 def corr2cov(corr, std):
-    '''convert correlation matrix to covariance matrix given standard deviation
+    """
+    convert correlation matrix to covariance matrix given standard deviation
 
     Parameters
     ----------
@@ -215,14 +281,16 @@ def corr2cov(corr, std):
     that multiplication is defined elementwise. np.ma.array are allowed, but
     not matrices.
 
-    '''
+    """
     corr = np.asanyarray(corr)
     std_ = np.asanyarray(std)
     cov = corr * np.outer(std_, std_)
     return cov
 
+
 def se_cov(cov):
-    '''get standard deviation from covariance matrix
+    """
+    get standard deviation from covariance matrix
 
     just a shorthand function np.sqrt(np.diag(cov))
 
@@ -235,6 +303,5 @@ def se_cov(cov):
     -------
     std : ndarray
         standard deviation from diagonal of cov
-
-    '''
+    """
     return np.sqrt(np.diag(cov))


Test patch:
diff --git a/statsmodels/stats/tests/test_moment_helpers.py b/statsmodels/stats/tests/test_moment_helpers.py
--- a/statsmodels/stats/tests/test_moment_helpers.py
+++ b/statsmodels/stats/tests/test_moment_helpers.py
@@ -4,19 +4,21 @@
 
 Author: Josef Perktold
 """
+import numpy as np
 import pytest
+from numpy.testing import assert_almost_equal, assert_, assert_equal
 
 from statsmodels.stats import moment_helpers
-from statsmodels.stats.moment_helpers import (cov2corr,
-                    mvsk2mc, mc2mvsk, mnc2mc, mc2mnc, cum2mc, mc2cum,
-                    mnc2cum)
+from statsmodels.stats.moment_helpers import (cov2corr, mvsk2mc, mc2mvsk,
+                                              mnc2mc, mc2mnc, cum2mc, mc2cum,
+                                              mnc2cum)
 
-import numpy as np
-from numpy.testing import assert_almost_equal, assert_, assert_equal
 
 def test_cov2corr():
-    cov_a = np.ones((3,3))+np.diag(np.arange(1,4)**2 - 1)
-    corr_a = np.array([[1, 1/2., 1/3.],[1/2., 1, 1/2./3.],[1/3., 1/2./3., 1]])
+    cov_a = np.ones((3, 3)) + np.diag(np.arange(1, 4) ** 2 - 1)
+    corr_a = np.array([[1, 1 / 2., 1 / 3.],
+                       [1 / 2., 1, 1 / 2. / 3.],
+                       [1 / 3., 1 / 2. / 3., 1]])
 
     corr = cov2corr(cov_a)
     assert_almost_equal(corr, corr_a, decimal=15)
@@ -32,9 +34,9 @@ def test_cov2corr():
 
     assert_(isinstance(corr_ma, np.ma.core.MaskedArray))
 
-    cov_ma2 = np.ma.array(cov_a, mask = [[False, True, False],
-                                         [True, False, False],
-                                         [False, False, False]])
+    cov_ma2 = np.ma.array(cov_a, mask=[[False, True, False],
+                                       [True, False, False],
+                                       [False, False, False]])
 
     corr_ma2 = cov2corr(cov_ma2)
     assert_(np.ma.allclose(corr_ma, corr, atol=1e-15))
@@ -48,7 +50,8 @@ def test_cov2corr():
       ([1.0, 1, 1, 4], [1.0, 1.0, 1.0, 1.0], [1.0, 0.0, 0.0, 3.0]),
       ([1.0, 2, 0, 3], [1.0, 2.0, 0.0, -9.0], [1.0, 1.0, -4.0, 9.0]),
       ([0.0, 2, 1, 3], [0.0, 2.0, 1.0, -9.0], [0.0, 2.0, 1.0, -9.0]),
-      ([1.0, 0.5, 0, 3], [1.0, 0.5, 0.0, 2.25], [1.0, -0.5, 0.5, 2.25]),  # neg.variance if mnc2<mnc1  # noqa:E501
+      # neg.variance if mnc2<mnc1
+      ([1.0, 0.5, 0, 3], [1.0, 0.5, 0.0, 2.25], [1.0, -0.5, 0.5, 2.25]),
       ([0.0, 0.5, 1, 3], [0.0, 0.5, 1.0, 2.25], [0.0, 0.5, 1.0, 2.25]),
       ([0.0, 1, 0, 3, 0], [0.0, 1.0, 0.0, 0.0, 0.0], [0., 1., 0., 0., 0.]),
       ([1.0, 1, 0, 3, 1], [1.0, 1.0, 0.0, 0.0, 1.0], [1., 0., -1., 6., -20.])]
@@ -86,6 +89,22 @@ def test_moment_conversion(mom):
         # assert_equal(mvsk2mnc(mc2mvsk(mom[0])), mc2mnc(mom[0]))
 
 
+rs = np.random.RandomState(12345)
+random_vals = np.random.randint(0, 100, 12).reshape(4, 3)
+multidimension_test_vals = [np.array([[5., 10., 1.],
+                                      [5., 10., 1.],
+                                      [5., 10., 1.],
+                                      [80., 310., 4.]]),
+                            random_vals]
+
+
+@pytest.mark.parametrize('test_vals', multidimension_test_vals)
+def test_multidimensional(test_vals):
+    assert_almost_equal(cum2mc(mnc2cum(mc2mnc(test_vals).T).T).T, test_vals)
+    assert_almost_equal(cum2mc(mc2cum(test_vals).T).T, test_vals)
+    assert_almost_equal(mvsk2mc(mc2mvsk(test_vals).T).T, test_vals)
+
+
 @pytest.mark.parametrize('func_name', ['cum2mc', 'cum2mc', 'mc2cum', 'mc2mnc',
                                        'mc2mvsk', 'mnc2cum', 'mnc2mc',
                                        'mnc2mc', 'mvsk2mc', 'mvsk2mnc'])
@@ -93,11 +112,15 @@ def test_moment_conversion_types(func_name):
     # written in 2009
     # TODO: why did I use list as return type?
     func = getattr(moment_helpers, func_name)
+
     assert (isinstance(func([1.0, 1, 0, 3]), list) or
-            isinstance(func(np.array([1.0, 1, 0, 3])), tuple))
+            isinstance(func(np.array([1.0, 1, 0, 3])), tuple) or
+            isinstance(func(np.array([1.0, 1, 0, 3])), np.ndarray))
 
     assert (isinstance(func(np.array([1.0, 1, 0, 3])), list) or
-            isinstance(func(np.array([1.0, 1, 0, 3])), tuple))
+            isinstance(func(np.array([1.0, 1, 0, 3])), tuple) or
+            isinstance(func(np.array([1.0, 1, 0, 3])), np.ndarray))
 
     assert (isinstance(func(tuple([1.0, 1, 0, 3])), list) or
-            isinstance(func(np.array([1.0, 1, 0, 3])), tuple))
+            isinstance(func(np.array([1.0, 1, 0, 3])), tuple) or
+            isinstance(func(np.array([1.0, 1, 0, 3])), np.ndarray))