Repository: statsmodels/statsmodels

Issue:
Lilliefors test produces p-value > 1
When I run

```
from statsmodels.stats.diagnostic import lilliefors
lilliefors(np.arange(5))
```

I get the output

`(0.13645537156723098, 2.1686565649378555)`

The second value is supposed to be the p-value. And p-values above 1 don't make any sense.


Discussion:

Resolving patch:
diff --git a/statsmodels/stats/_lilliefors.py b/statsmodels/stats/_lilliefors.py
--- a/statsmodels/stats/_lilliefors.py
+++ b/statsmodels/stats/_lilliefors.py
@@ -1,5 +1,13 @@
 # -*- coding: utf-8 -*-
 """
+Implements Lilliefors corrected Kolmogorov-Smirnov tests for normal and
+exponential distributions.
+
+`kstest_fit` is provided as a top-level function to access both tests.
+`kstest_normal` and `kstest_exponential` are provided as convenience functions
+with the appropriate test as the default.
+`lilliefors` is provided as an alias for `kstest_fit`.
+
 Created on Sat Oct 01 13:16:49 2011
 
 Author: Josef Perktold
@@ -15,11 +23,23 @@
 Hubert W. Lilliefors
 Journal of the American Statistical Association, Vol. 62, No. 318. (Jun., 1967), pp. 399-402.
 
+
+---
+
+Updated 2017-07-23
+Jacob C. Kimmel
+
+Ref:
+Lilliefors, H.W.
+On the Kolmogorov-Smirnov test for the exponential distribution with mean unknown.
+Journal of the American Statistical Association, Vol 64, No. 325. (1969), pp. 387–389.
 """
 from statsmodels.compat.python import string_types
 import numpy as np
 from scipy.interpolate import interp1d
 from scipy import stats
+from .tabledist import TableDist
+
 
 def ksstat(x, cdf, alternative='two_sided', args=()):
     """
@@ -87,55 +107,130 @@ def ksstat(x, cdf, alternative='two_sided', args=()):
             return Dmin
 
     D = np.max([Dplus,Dmin])
+
     return D
 
 
-#new version with tabledist
-#--------------------------
+# new version with tabledist
+# --------------------------
+
+def get_lilliefors_table(dist='norm'):
+    '''
+    Generates tables for significance levels of Lilliefors test statistics
+
+    Tables for available normal and exponential distribution testing,
+    as specified in Lilliefors references above
 
-def get_lilliefors_table():
-    #function just to keep things together
-    from .tabledist import TableDist
-    #for this test alpha is sf probability, i.e. right tail probability
+    Parameters
+    ----------
+    dist : string.
+        distribution being tested in set {'norm', 'exp'}.
 
-    alpha = np.array([ 0.2  ,  0.15 ,  0.1  ,  0.05 ,  0.01 ,  0.001])[::-1]
-    size = np.array([ 4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,  15,
+    Returns
+    -------
+    lf : TableDist object.
+        table of critical values
+    '''
+    # function just to keep things together
+    # for this test alpha is sf probability, i.e. right tail probability
+
+    if dist == 'norm':
+        alpha = np.array([ 0.2  ,  0.15 ,  0.1  ,  0.05 ,  0.01 ,  0.001])[::-1]
+        size = np.array([ 4,   5,   6,   7,   8,   9,  10,  11,  12,  13,  14,  15,
                      16,  17,  18,  19,  20,  25,  30,  40, 100, 400, 900], float)
 
-    #critical values, rows are by sample size, columns are by alpha
-    crit_lf = np.array(   [[303, 321, 346, 376, 413, 433],
-                           [289, 303, 319, 343, 397, 439],
-                           [269, 281, 297, 323, 371, 424],
-                           [252, 264, 280, 304, 351, 402],
-                           [239, 250, 265, 288, 333, 384],
-                           [227, 238, 252, 274, 317, 365],
-                           [217, 228, 241, 262, 304, 352],
-                           [208, 218, 231, 251, 291, 338],
-                           [200, 210, 222, 242, 281, 325],
-                           [193, 202, 215, 234, 271, 314],
-                           [187, 196, 208, 226, 262, 305],
-                           [181, 190, 201, 219, 254, 296],
-                           [176, 184, 195, 213, 247, 287],
-                           [171, 179, 190, 207, 240, 279],
-                           [167, 175, 185, 202, 234, 273],
-                           [163, 170, 181, 197, 228, 266],
-                           [159, 166, 176, 192, 223, 260],
-                           [143, 150, 159, 173, 201, 236],
-                           [131, 138, 146, 159, 185, 217],
-                           [115, 120, 128, 139, 162, 189],
-                           [ 74,  77,  82,  89, 104, 122],
-                           [ 37,  39,  41,  45,  52,  61],
-                           [ 25,  26,  28,  30,  35,  42]])[:,::-1] / 1000.
-
-
-    lf = TableDist(alpha, size, crit_lf)
+        # critical values, rows are by sample size, columns are by alpha
+        crit_lf = np.array(   [[303, 321, 346, 376, 413, 433],
+                               [289, 303, 319, 343, 397, 439],
+                               [269, 281, 297, 323, 371, 424],
+                               [252, 264, 280, 304, 351, 402],
+                               [239, 250, 265, 288, 333, 384],
+                               [227, 238, 252, 274, 317, 365],
+                               [217, 228, 241, 262, 304, 352],
+                               [208, 218, 231, 251, 291, 338],
+                               [200, 210, 222, 242, 281, 325],
+                               [193, 202, 215, 234, 271, 314],
+                               [187, 196, 208, 226, 262, 305],
+                               [181, 190, 201, 219, 254, 296],
+                               [176, 184, 195, 213, 247, 287],
+                               [171, 179, 190, 207, 240, 279],
+                               [167, 175, 185, 202, 234, 273],
+                               [163, 170, 181, 197, 228, 266],
+                               [159, 166, 176, 192, 223, 260],
+                               [143, 150, 159, 173, 201, 236],
+                               [131, 138, 146, 159, 185, 217],
+                               [115, 120, 128, 139, 162, 189],
+                               [ 74,  77,  82,  89, 104, 122],
+                               [ 37,  39,  41,  45,  52,  61],
+                               [ 25,  26,  28,  30,  35,  42]])[:,::-1] / 1000.
+
+
+        # also build a table for larger sample sizes
+        def f(n):
+            return np.array([0.736, 0.768, 0.805, 0.886, 1.031]) / np.sqrt(n)
+
+        higher_sizes = np.array([35, 40, 45, 50, 60, 70,
+                                 80, 100, 200, 500, 1000,
+                                 2000, 3000, 5000, 10000, 100000], float)
+        higher_crit_lf = np.zeros([higher_sizes.shape[0], crit_lf.shape[1]-1])
+        for i in range(len(higher_sizes)):
+            higher_crit_lf[i, :] = f(higher_sizes[i])
+
+        alpha_large = alpha[:-1]
+        size_large = np.concatenate([size, higher_sizes])
+        crit_lf_large = np.vstack([crit_lf[:-4,:-1], higher_crit_lf])
+        lf = TableDist(alpha, size, crit_lf)
+
+    elif dist == 'exp':
+        alpha = np.array([0.2,  0.15,  0.1,  0.05, 0.01])[::-1]
+        size = np.array([3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,25,30],
+                        float)
+
+        crit_lf = np.array([   [451, 479, 511, 551, 600],
+                                [396, 422, 499, 487, 548],
+                                [359, 382, 406, 442, 504],
+                                [331, 351, 375, 408, 470],
+                                [309, 327, 350, 382, 442],
+                                [291, 308, 329, 360, 419],
+                                [277, 291, 311, 341, 399],
+                                [263, 277, 295, 325, 380],
+                                [251, 264, 283, 311, 365],
+                                [241, 254, 271, 298, 351],
+                                [232, 245, 261, 287, 338],
+                                [224, 237, 252, 277, 326],
+                                [217, 229, 244, 269, 315],
+                                [211, 222, 236, 261, 306],
+                                [204, 215, 229, 253, 297],
+                                [199, 210, 223, 246, 289],
+                                [193, 204, 218, 239, 283],
+                                [188, 199, 212, 234, 278],
+                                [170, 180, 191, 210, 247],
+                                [155, 164, 174, 192, 226]])[:,::-1] / 1000.
+
+        def f(n):
+            return np.array([.86, .91, .96, 1.06, 1.25]) / np.sqrt(n)
+
+        higher_sizes = np.array([35, 40, 45, 50, 60, 70,
+                                80, 100, 200, 500, 1000,
+                                2000, 3000, 5000, 10000, 100000], float)
+        higher_crit_lf = np.zeros([higher_sizes.shape[0], crit_lf.shape[1]])
+        for i in range(len(higher_sizes)):
+            higher_crit_lf[i,:] = f(higher_sizes[i])
+
+        size = np.concatenate([size, higher_sizes])
+        crit_lf = np.vstack([crit_lf, higher_crit_lf])
+        lf = TableDist(alpha, size, crit_lf)
+    else:
+        raise ValueError("Invalid dist parameter. dist must be 'norm' or 'exp'")
 
     return lf
 
-lilliefors_table = get_lilliefors_table()
+lilliefors_table_norm = get_lilliefors_table(dist='norm')
+lilliefors_table_expon = get_lilliefors_table(dist='exp')
+
 
 def pval_lf(Dmax, n):
-    '''approximate pvalues for Lilliefors test of normality
+    '''approximate pvalues for Lilliefors test
 
     This is only valid for pvalues smaller than 0.1 which is not checked in
     this function.
@@ -169,17 +264,17 @@ def pval_lf(Dmax, n):
     '''
 
     #todo: check boundaries, valid range for n and Dmax
-    if n>100:
-        Dmax *= (n/100.)**0.49
+    if n > 100:
+        Dmax *= (n / 100.)**0.49
         n = 100
-    pval = np.exp(-7.01256*Dmax**2 *(n + 2.78019)
-                + 2.99587 * Dmax * np.sqrt(n + 2.78019) - 0.122119
-                + 0.974598/np.sqrt(n) + 1.67997/n)
+    pval = np.exp(-7.01256 * Dmax**2 * (n + 2.78019)
+                  + 2.99587 * Dmax * np.sqrt(n + 2.78019) - 0.122119
+                  + 0.974598/np.sqrt(n) + 1.67997/n)
     return pval
 
 
-def kstest_normal(x, pvalmethod='approx'):
-    '''lilliefors test for normality,
+def kstest_fit(x, dist='norm', pvalmethod='approx'):
+    '''lilliefors test for normality or an exponential distribution.
 
     Kolmogorov Smirnov test with estimated mean and variance
 
@@ -187,15 +282,26 @@ def kstest_normal(x, pvalmethod='approx'):
     ----------
     x : array_like, 1d
         data series, sample
+    dist : string.
+        distribution to test in set {'norm', 'exp'}.
     pvalmethod : 'approx', 'table'
+        'approx' is only valid for normality. if `dist = 'exp'`,
+        `table` is returned.
         'approx' uses the approximation formula of Dalal and Wilkinson,
         valid for pvalues < 0.1. If the pvalue is larger than 0.1, then the
         result of `table` is returned
+
+        For normality:
         'table' uses the table from Dalal and Wilkinson, which is available
         for pvalues between 0.001 and 0.2, and the formula of Lilliefors for
         large n (n>900). Values in the table are linearly interpolated.
         Values outside the range will be returned as bounds, 0.2 for large and
         0.001 for small pvalues.
+        For exponential:
+        'table' uses the table from Lilliefors 1967, available for pvalues
+        between 0.01 and 0.2.
+        Values outside the range will be returned as bounds, 0.2 for large and
+        0.01 for small pvalues.
 
     Returns
     -------
@@ -216,31 +322,48 @@ def kstest_normal(x, pvalmethod='approx'):
     '''
 
     x = np.asarray(x)
-    z = (x-x.mean())/x.std(ddof=1)
-    nobs = len(z)
+    nobs = len(x)
+
+    if dist == 'norm':
+        z = (x - x.mean()) / x.std(ddof=1)
+        test_d = stats.norm.cdf
+        lilliefors_table = lilliefors_table_norm
+    elif dist == 'exp':
+        z = x / x.mean()
+        test_d = stats.expon.cdf
+        lilliefors_table = lilliefors_table_expon
+        pvalmethod = 'table'
+    else:
+        raise ValueError("Invalid dist parameter. dist must be 'norm' or 'exp'")
 
-    d_ks = ksstat(z, stats.norm.cdf, alternative='two_sided')
+    d_ks = ksstat(z, test_d, alternative='two_sided')
 
     if pvalmethod == 'approx':
         pval = pval_lf(d_ks, nobs)
+        # check pval is in desired range
+        if pval > 0.1:
+            pval = lilliefors_table.prob(d_ks, nobs)
     elif pvalmethod == 'table':
-        #pval = pval_lftable(d_ks, nobs)
         pval = lilliefors_table.prob(d_ks, nobs)
 
     return d_ks, pval
 
 
-lilliefors = kstest_normal
 
+lilliefors = kstest_fit
 lillifors = np.deprecate(lilliefors, 'lillifors', 'lilliefors',
                                "Use lilliefors, lillifors will be "
                                "removed in 0.9 \n(Note: misspelling missing 'e')")
 
+# namespace aliases
+from functools import partial
+kstest_normal = kstest_fit
+kstest_exponential = partial(kstest_fit, dist='exp')
 
-#old version:
-#------------
-
-tble = '''\
+# old version:
+# ------------
+'''
+tble = \
 00 20 15 10 05 01 .1
 4 .303 .321 .346 .376 .413 .433
 5 .289 .303 .319 .343 .397 .439
diff --git a/statsmodels/stats/diagnostic.py b/statsmodels/stats/diagnostic.py
--- a/statsmodels/stats/diagnostic.py
+++ b/statsmodels/stats/diagnostic.py
@@ -9,5 +9,5 @@
     het_breushpagan, acorr_breush_godfrey  # deprecated because of misspelling
     )
 
-from ._lilliefors import kstest_normal, lilliefors, lillifors # lillifors is deprecated
+from ._lilliefors import kstest_fit, lilliefors, lillifors # lillifors is deprecated
 from ._adnorm import normal_ad


Test patch:
diff --git a/statsmodels/stats/tests/test_diagnostic.py b/statsmodels/stats/tests/test_diagnostic.py
--- a/statsmodels/stats/tests/test_diagnostic.py
+++ b/statsmodels/stats/tests/test_diagnostic.py
@@ -555,7 +555,7 @@ def test_normality(self):
         #> lt = lillie.test(residuals(fm)[1:20])
         #> mkhtest(lt, "lilliefors", "-")
         lilliefors3 = dict(statistic=0.1333956004203103,
-                          pvalue=0.4618672180799566, parameters=(), distr='-')
+                          pvalue=0.20, parameters=(), distr='-')
 
         lf1 = smsdia.lilliefors(res.resid)
         lf2 = smsdia.lilliefors(res.resid**2)
@@ -955,4 +955,3 @@ def test_outlier_test():
     Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
 
     '''
-
diff --git a/statsmodels/stats/tests/test_lilliefors.py b/statsmodels/stats/tests/test_lilliefors.py
new file mode 100644
--- /dev/null
+++ b/statsmodels/stats/tests/test_lilliefors.py
@@ -0,0 +1,59 @@
+
+import numpy as np
+from numpy.testing import assert_almost_equal
+from scipy import stats
+
+from statsmodels.stats._lilliefors import lilliefors
+
+
+class TestLilliefors(object):
+
+    def test_normal(self):
+        np.random.seed(3975)
+        x_n = stats.norm.rvs(size=500)
+
+        # R function call:
+        # require(nortest)
+        # lillie.test(x_n)
+
+        d_ks_norm, p_norm = lilliefors(x_n, dist='norm')
+        # shift normal distribution > 0 to exactly mirror R `KScorrect` test
+        # R `KScorrect` requires all values tested for exponential
+        # distribution to be > 0
+        # R function call:
+        # require(KScorrect)
+        # LcKS(x_n+abs(min(x_n))+0.001, 'pexp')
+
+        d_ks_exp, p_exp = lilliefors(x_n+np.abs(x_n.min()) + 0.001, dist='exp')
+        # assert normal
+        assert_almost_equal(d_ks_norm, 0.025957, decimal=3)
+        assert_almost_equal(p_norm, 0.2000, decimal=3)
+        # assert exp
+        assert_almost_equal(d_ks_exp, 0.3436007, decimal=3)
+        assert_almost_equal(p_exp, 0.01, decimal=3)
+
+    def test_expon(self):
+        np.random.seed(3975)
+        x_e = stats.expon.rvs(size=500)
+        # R function call:
+        # require(nortest)
+        # lillie.test(x_n)
+        d_ks_norm, p_norm = lilliefors(x_e, dist='norm')
+        # R function call:
+        # require(KScorrect)
+        # LcKS(x_e, 'pexp')
+        d_ks_exp, p_exp = lilliefors(x_e, dist='exp')
+        # assert normal
+        assert_almost_equal(d_ks_norm, 0.15581, decimal=3)
+        assert_almost_equal(p_norm, 2.2e-16, decimal=3)
+        # assert exp
+        assert_almost_equal(d_ks_exp, 0.02763748, decimal=3)
+        assert_almost_equal(p_exp, 0.200, decimal=3)
+
+    def test_pval_bounds(self):
+        x = np.arange(1,10)
+        d_ks_n, p_n = lilliefors(x, dist='norm')
+        d_ks_e, p_e = lilliefors(x, dist='exp')
+
+        assert_almost_equal(p_n, 0.200, decimal=7)
+        assert_almost_equal(p_e, 0.200, decimal=7)