Repository: scipy/scipy

Issue:
anderson_ksamp p-values>1
I've been using the anderson_ksamp function on two data sets. I find that occasionally I get a p-value>1; which makes no sense. I don't know if this is a bug with the scipy function, or a statistical error.

For example, for a p-value of 1.07, the two lists I am using are:

[0.39999999999999858, 0.5, 1.5, 0.60000000000000142, 0.30000000000000071, 0.30000000000000071, 0.69999999999999929, 0.89999999999999858, 0.80000000000000071, 0.69999999999999929, 0.69999999999999929, 0.30000000000000071, 1.1000000000000014, 0.60000000000000142, 0.5, 0.39999999999999858, 1.3999999999999986, 1.0, 1.0, 0.39999999999999858, 1.3999999999999986, 0.5, 0.69999999999999929, 0.60000000000000142, 1.3000000000000007, 0.39999999999999858, 0.39999999999999858, 0.89999999999999858, 0.5, 1.1999999999999993, 1.3000000000000007, 1.5, 0.89999999999999858, 1.0, 0.69999999999999929, 1.3999999999999986, 0.69999999999999929, 0.89999999999999858, 0.60000000000000142, 0.60000000000000142, 0.39999999999999858]

...and...

[0.30000000000000071, 1.3200000000000003, 1.0500000000000007, 0.60000000000000142, -20.0, 0.5, 0.98999999999999844, 0.44999999999999929, 0.64999999999999858, 0.48000000000000043, 1.4499999999999993, 0.67999999999999972, 1.0, 0.69999999999999929, 1.5399999999999991, 0.32000000000000028, 1.0, 2.0, 1.25, 0.5, 1.3500000000000014, 1.5, 0.5, 0.85000000000000142, 0.5, 0.80000000000000071, 0.64999999999999858, 0.5, 0.30000000000000071, 0.35000000000000142, 0.5, 0.42999999999999972, 1.0, 0.69999999999999929, 0.94000000000000128, 0.30000000000000071, 0.42000000000000171, 0.62999999999999901, 0.44999999999999929, 0.53999999999999915, 0.69999999999999929, 1.620000000000001, 1.1999999999999993, 0.35000000000000142, 0.5, 0.75, 1.2600000000000016, 0.69999999999999929, 0.69999999999999929, 0.92000000000000171, 0.55000000000000071, 0.76000000000000156, 1.1499999999999986, 0.44000000000000128, 0.64999999999999858, 0.5, 1.3999999999999986, 1.25, 1.6499999999999986, 0.69999999999999929, 0.94999999999999929, 0.37999999999999901, 0.5, 1.3500000000000014, 0.30000000000000071, 0.5, 1.370000000000001, 0.69999999999999929, 0.71999999999999886, 0.89999999999999858, 1.5, 0.75, 0.44999999999999929, 0.75, 0.39999999999999858, 0.5, 0.39999999999999858, 0.35999999999999943, 0.55000000000000071, 0.35000000000000142, 0.48000000000000043, 0.89999999999999858, 0.85000000000000142, 1.1999999999999993, 0.55000000000000071, 0.35000000000000142, 0.64999999999999858, 0.89999999999999858, 0.80000000000000071, 0.64999999999999858, 0.55000000000000071, 1.0500000000000007, 0.5, 0.80000000000000071, 0.64999999999999858, 1.3000000000000007, 1.0, 1.0, 0.64999999999999858, 0.94999999999999929, 1.4100000000000001, 0.80000000000000071, 1.0, 0.55000000000000071, 0.85000000000000142, 0.85000000000000142, 0.33999999999999986, 0.64999999999999858, 0.94999999999999929, 1.1000000000000014, 0.64999999999999858, 0.69999999999999929, 0.55000000000000071, 0.37999999999999901, 1.3000000000000007, 0.67000000000000171, 0.69999999999999929, 1.0, 0.60000000000000142, 0.82999999999999829, 0.89999999999999858, 1.3000000000000007, 1.3000000000000007, 1.1499999999999986, 0.69999999999999929, 0.85000000000000142, 0.60000000000000142, 1.0, 0.60000000000000142, 1.3000000000000007, 0.44999999999999929, 1.1499999999999986, 0.30000000000000071, 1.3500000000000014, 1.129999999999999, 0.33999999999999986, 0.30000000000000071, 1.4100000000000001, 0.69999999999999929, 1.25, 0.5, 0.44000000000000128, 1.1600000000000001, 0.5, 0.53000000000000114, 0.55000000000000071, 0.44999999999999929, 0.92999999999999972, 1.3900000000000006, 0.85000000000000142, 0.30000000000000071, 0.30000000000000071, 0.39999999999999858, 0.64999999999999858, 1.620000000000001, 0.64999999999999858, 1.1999999999999993, 0.35000000000000142, 0.80000000000000071, 0.30000000000000071, -20.0, 1.1999999999999993, 0.78000000000000114, 0.30000000000000071, 0.60000000000000142, 1.1999999999999993, 0.80000000000000071, 1.3500000000000014, 1.6999999999999993, 1.4499999999999993, 0.62999999999999901, 0.83999999999999986, 0.30000000000000071, 0.67000000000000171, 0.57999999999999829, 0.64999999999999858, 0.75, 0.69000000000000128, 1.1000000000000014, 0.64999999999999858, 1.25, 1.6999999999999993, 0.60000000000000142, 0.89000000000000057, 1.75, 1.3900000000000006, 0.30000000000000071, 1.1000000000000014, 0.44999999999999929, 0.30000000000000071, 0.64999999999999858, 1.3399999999999999, 0.30000000000000071, 0.39999999999999858, 0.80000000000000071, 1.6999999999999993, 0.30000000000000071, 0.64999999999999858, 0.78000000000000114, 0.58999999999999986, 1.8500000000000014, 1.4499999999999993, 0.5, 1.7399999999999984, 0.75, 1.1400000000000006, 1.1000000000000014, 0.37999999999999901, 1.3599999999999994, 1.4499999999999993, 0.5, 0.30000000000000071, 0.69999999999999929, 1.0, 0.69999999999999929, 0.98000000000000043, 0.82999999999999829, 0.60000000000000142, 0.87999999999999901, 0.37000000000000099, 0.5, 0.92999999999999972, 0.35000000000000142, 0.55000000000000071, 0.39999999999999858, 1.0500000000000007, 0.35000000000000142, 0.89999999999999858, 0.64999999999999858, 1.5199999999999996, 1.25, 0.69999999999999929, 1.5, 0.53000000000000114, 0.39999999999999858, 0.80000000000000071, -20.0, -20.0, 0.30000000000000071, 0.64999999999999858, 1.1799999999999997, 0.78000000000000114, 0.64999999999999858, 0.39000000000000057, 0.60000000000000142, 0.75, 0.44999999999999929, 0.69999999999999929, 0.67999999999999972, 0.64999999999999858, 0.48000000000000043, 0.48000000000000043, 1.75, 0.60000000000000142, 1.0, 0.37000000000000099, 0.69999999999999929, 0.35000000000000142, 0.60000000000000142, 1.1000000000000014, 2.3999999999999986, 0.67000000000000171, 0.69999999999999929, 0.44999999999999929, 0.55000000000000071, 0.60000000000000142, 1.1999999999999993, 0.5, 1.1000000000000014, 0.30000000000000071, 0.89999999999999858, 0.55000000000000071, 0.80000000000000071, 1.0500000000000007, 0.71999999999999886, 0.30000000000000071, 0.73999999999999844, 0.64999999999999858, 0.98000000000000043, 1.3999999999999986, 0.94999999999999929, 0.60000000000000142, 0.69999999999999929, 0.35000000000000142, 0.64999999999999858, 1.5, 0.89999999999999858, 1.1000000000000014, 1.75, 0.35000000000000142, 0.35999999999999943, 0.5, 1.25, 1.6999999999999993, 0.39999999999999858, 0.51999999999999957, 0.30000000000000071, 1.0899999999999999, 0.39999999999999858, 1.5399999999999991, 1.6999999999999993, 0.85000000000000142, 0.64999999999999858, 1.0, 0.51999999999999957, 0.5, 1.2600000000000016, 1.1499999999999986, 1.3000000000000007, 0.39999999999999858, 1.1000000000000014, 0.48000000000000043, 0.94999999999999929, 0.69999999999999929, 0.33999999999999986, 0.30000000000000071, 0.60000000000000142, 0.39999999999999858, 0.85000000000000142, 1.1499999999999986, 0.98000000000000043, 0.80000000000000071, 0.60000000000000142, 0.69999999999999929, 0.60000000000000142, 0.30000000000000071, -20.0, 0.35000000000000142, 0.96000000000000085, 1.3500000000000014, 1.0, 1.1499999999999986, 0.75, 1.0, 0.64999999999999858, 0.60000000000000142, 0.44999999999999929, 0.94999999999999929, 0.85000000000000142, 0.39999999999999858, 0.55000000000000071, 0.35999999999999943, 0.46999999999999886, 0.55000000000000071, 0.62999999999999901, 0.30000000000000071, 1.0, 0.60000000000000142, 0.53000000000000114, 0.64999999999999858, 0.60000000000000142, 0.30000000000000071, 0.30000000000000071, 0.67000000000000171, 2.0500000000000007, 0.85000000000000142, 0.94999999999999929, 0.64999999999999858, 0.60000000000000142, 0.98000000000000043, 1.1000000000000014, 0.30000000000000071, 0.39999999999999858, 1.5399999999999991, 0.69999999999999929, 0.30000000000000071, 0.35000000000000142, 1.0, 0.35999999999999943, 0.46999999999999886, 0.64999999999999858, 0.48999999999999844, 0.64999999999999858, 0.60000000000000142, 1.3000000000000007, 0.85000000000000142, 1.379999999999999, 0.67999999999999972, 0.69999999999999929, 0.94999999999999929, 0.69999999999999929, 0.94999999999999929, 0.55000000000000071, 1.4499999999999993, 0.30000000000000071, 0.64999999999999858, 1.3000000000000007, 0.78000000000000114, 0.55000000000000071, 0.30000000000000071]



Discussion:
The pvalues should be clipped to be in [0,1].

However, there is a warning that the pvalues are just obtained by extrapolation. The p-values will not be very accurate outside the range of the available table values 0.25, 0.1, 0.05, 0.025, 0.01

(One possibility, eventually, is to simulate our own critical values for some of these tables to expand the range and improve the approximation. 
Specifically, for AD 2samp, ksample there is an algorithm, IIRC) 

Is it possible to just use the calculated values from the R [`kSamples`](https://cran.r-project.org/web/packages/kSamples/kSamples.pdf) package?

I came across this as well, any chance to update the tables and/or mitigate the extreme extrapolation?

http://stats.stackexchange.com/questions/225588/why-does-scipy-stats-anderson-ksamp-give-a-p-value-of-over-a-million-for-these-d?atw=1
I came across this problem too. Should we just chop it to 1 when it is over 1 ?
@cclinus This may not fix the problem properly though. There's likely some sort of extrapolation error, so there's a reasonable chance that a lot of the p-values less than 1 are wrong too.

Resolving patch:
diff --git a/scipy/stats/morestats.py b/scipy/stats/morestats.py
--- a/scipy/stats/morestats.py
+++ b/scipy/stats/morestats.py
@@ -1568,7 +1568,8 @@ def anderson_ksamp(samples, midrank=True):
         The critical values for significance levels 25%, 10%, 5%, 2.5%, 1%.
     significance_level : float
         An approximate significance level at which the null hypothesis for the
-        provided samples can be rejected.
+        provided samples can be rejected. The value is floored / capped at
+        1% / 25%.
 
     Raises
     ------
@@ -1583,7 +1584,7 @@ def anderson_ksamp(samples, midrank=True):
 
     Notes
     -----
-    [1]_ Defines three versions of the k-sample Anderson-Darling test:
+    [1]_ defines three versions of the k-sample Anderson-Darling test:
     one for continuous distributions and two for discrete
     distributions, in which ties between samples may occur. The
     default of this routine is to compute the version based on the
@@ -1594,6 +1595,11 @@ def anderson_ksamp(samples, midrank=True):
     only slightly if a few collisions due to round-off errors occur in
     the test not adjusted for ties between samples.
 
+    The critical values are taken from [1]_. p-values are floored / capped
+    at 1% / 25%. Since the range of criticl values might be extended in
+    future releases, it is recommended not to test ``p == 0.01``, but rather
+    ``p <= 0.01`` (analogously for the upper bound).
+
     .. versionadded:: 0.14.0
 
     References
@@ -1621,14 +1627,15 @@ def anderson_ksamp(samples, midrank=True):
 
 
     The null hypothesis cannot be rejected for three samples from an
-    identical distribution. The approximate p-value (87%) has to be
-    computed by extrapolation and may not be very accurate:
+    identical distribution. The reported p-value (25%) has been capped and
+    may not be very accurate (since it corresponds to the value 0.449
+    whereas the statistic is -0.731):
 
     >>> stats.anderson_ksamp([np.random.normal(size=50),
     ... np.random.normal(size=30), np.random.normal(size=20)])
     (-0.73091722665244196,
       array([ 0.44925884,  1.3052767 ,  1.9434184 ,  2.57696569,  3.41634856]),
-      0.8789283903979661)
+      0.25)
 
     """
     k = len(samples)
@@ -1672,13 +1679,21 @@ def anderson_ksamp(samples, midrank=True):
     b1 = np.array([-0.245, 0.25, 0.678, 1.149, 1.822])
     b2 = np.array([-0.105, -0.305, -0.362, -0.391, -0.396])
     critical = b0 + b1 / math.sqrt(m) + b2 / m
-    pf = np.polyfit(critical, log(np.array([0.25, 0.1, 0.05, 0.025, 0.01])), 2)
-    if A2 < critical.min() or A2 > critical.max():
-        warnings.warn("approximate p-value will be computed by extrapolation")
-    try:
+
+    sig = np.array([0.25, 0.1, 0.05, 0.025, 0.01])
+    if A2 < critical.min():
+        p = sig.max()
+        warnings.warn("p-value capped: true value larger than {}".format(p),
+                      stacklevel=2)
+    elif A2 > critical.max():
+        p = sig.min()
+        warnings.warn("p-value floored: true value smaller than {}".format(p),
+                      stacklevel=2)
+    else:
+        # interpolation of probit of significance level
+        pf = np.polyfit(critical, log(sig), 2)
         p = math.exp(np.polyval(pf, A2))
-    except (OverflowError,):
-        p = float("inf")
+
     return Anderson_ksampResult(A2, critical, p)
 
 


Test patch:
diff --git a/scipy/stats/tests/test_morestats.py b/scipy/stats/tests/test_morestats.py
--- a/scipy/stats/tests/test_morestats.py
+++ b/scipy/stats/tests/test_morestats.py
@@ -270,13 +270,13 @@ def test_example1a(self):
         assert_warns(UserWarning, stats.anderson_ksamp, (t1, t2, t3, t4),
                      midrank=False)
         with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
+            sup.filter(UserWarning, message='p-value floored')
             Tk, tm, p = stats.anderson_ksamp((t1, t2, t3, t4), midrank=False)
 
         assert_almost_equal(Tk, 4.449, 3)
         assert_array_almost_equal([0.4985, 1.3237, 1.9158, 2.4930, 3.2459],
                                   tm, 4)
-        assert_almost_equal(p, 0.0021, 4)
+        assert_equal(p, 0.01)  # floor at 0.01; in technical report p = 0.0021
 
     def test_example1b(self):
         # Example data from Scholz & Stephens (1987), originally
@@ -288,13 +288,13 @@ def test_example1b(self):
         t3 = np.array([34.0, 35.0, 39.0, 40.0, 43.0, 43.0, 44.0, 45.0])
         t4 = np.array([34.0, 34.8, 34.8, 35.4, 37.2, 37.8, 41.2, 42.8])
         with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
+            sup.filter(UserWarning, message='p-value floored')
             Tk, tm, p = stats.anderson_ksamp((t1, t2, t3, t4), midrank=True)
 
         assert_almost_equal(Tk, 4.480, 3)
         assert_array_almost_equal([0.4985, 1.3237, 1.9158, 2.4930, 3.2459],
                                   tm, 4)
-        assert_almost_equal(p, 0.0020, 4)
+        assert_equal(p, 0.01)  # floor at 0.01; in technical report p = 0.0020
 
     def test_example2a(self):
         # Example data taken from an earlier technical report of
@@ -320,7 +320,7 @@ def test_example2a(self):
         t14 = [102, 209, 14, 57, 54, 32, 67, 59, 134, 152, 27, 14, 230, 66,
                61, 34]
         with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
+            sup.filter(UserWarning, message='p-value floored')
             Tk, tm, p = stats.anderson_ksamp((t1, t2, t3, t4, t5, t6, t7, t8,
                                               t9, t10, t11, t12, t13, t14),
                                              midrank=False)
@@ -328,7 +328,7 @@ def test_example2a(self):
         assert_almost_equal(Tk, 3.288, 3)
         assert_array_almost_equal([0.5990, 1.3269, 1.8052, 2.2486, 2.8009],
                                   tm, 4)
-        assert_almost_equal(p, 0.0041, 4)
+        assert_equal(p, 0.01)  # floor at 0.01; in technical report p = 0.0041
 
     def test_example2b(self):
         # Example data taken from an earlier technical report of
@@ -353,7 +353,7 @@ def test_example2b(self):
         t14 = [102, 209, 14, 57, 54, 32, 67, 59, 134, 152, 27, 14, 230, 66,
                61, 34]
         with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
+            sup.filter(UserWarning, message='p-value floored')
             Tk, tm, p = stats.anderson_ksamp((t1, t2, t3, t4, t5, t6, t7, t8,
                                               t9, t10, t11, t12, t13, t14),
                                              midrank=True)
@@ -361,7 +361,70 @@ def test_example2b(self):
         assert_almost_equal(Tk, 3.294, 3)
         assert_array_almost_equal([0.5990, 1.3269, 1.8052, 2.2486, 2.8009],
                                   tm, 4)
-        assert_almost_equal(p, 0.0041, 4)
+        assert_equal(p, 0.01)  # floor at 0.01; in technical report p = 0.0041
+
+    def test_R_kSamples(self):
+        # test values generates with R package kSamples
+        # package version 1.2-6 (2017-06-14)
+        # r1 = 1:100
+        # continuous case (no ties) --> version  1
+        # res <- kSamples::ad.test(r1, r1 + 40.5)
+        # res$ad[1, "T.AD"] #  41.105
+        # res$ad[1, " asympt. P-value"] #  5.8399e-18
+        #
+        # discrete case (ties allowed) --> version  2 (here: midrank=True)
+        # res$ad[2, "T.AD"] #  41.235
+        #
+        # res <- kSamples::ad.test(r1, r1 + .5)
+        # res$ad[1, "T.AD"] #  -1.2824
+        # res$ad[1, " asympt. P-value"] #  1
+        # res$ad[2, "T.AD"] #  -1.2944
+        #
+        # res <- kSamples::ad.test(r1, r1 + 7.5)
+        # res$ad[1, "T.AD"] # 1.4923
+        # res$ad[1, " asympt. P-value"] # 0.077501
+        #
+        # res <- kSamples::ad.test(r1, r1 + 6)
+        # res$ad[2, "T.AD"] # 0.63892
+        # res$ad[2, " asympt. P-value"] # 0.17981
+
+        x1 = np.linspace(1, 100, 100)
+        # test case: different distributions;p-value floored at 0.01
+        # test case for issue #5493 / #8536
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, message='p-value floored')
+            s, _, p = stats.anderson_ksamp([x1, x1 + 40.5], midrank=False)
+        assert_almost_equal(s, 41.105, 3)
+        assert_equal(p, 0.01)
+
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, message='p-value floored')
+            s, _, p = stats.anderson_ksamp([x1, x1 + 40.5])
+        assert_almost_equal(s, 41.235, 3)
+        assert_equal(p, 0.01)
+
+        # test case: similar distributions --> p-value capped at 0.25
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, message='p-value capped')
+            s, _, p = stats.anderson_ksamp([x1, x1 + .5], midrank=False)
+        assert_almost_equal(s, -1.2824, 4)
+        assert_equal(p, 0.25)
+
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, message='p-value capped')
+            s, _, p = stats.anderson_ksamp([x1, x1 + .5])
+        assert_almost_equal(s, -1.2944, 4)
+        assert_equal(p, 0.25)
+
+        # test case: check interpolated p-value in range [0.01, 0.25] (no ties)
+        s, _, p = stats.anderson_ksamp([x1, x1 + 7.5], midrank=False)
+        assert_almost_equal(s, 1.4923, 4)
+        assert_allclose(p, 0.0775, atol=0.005, rtol=0)
+
+        # test case: check interpolated p-value in range [0.01, 0.25] (w/ ties)
+        s, _, p = stats.anderson_ksamp([x1, x1 + 6])
+        assert_almost_equal(s, 0.6389, 4)
+        assert_allclose(p, 0.1798, atol=0.005, rtol=0)
 
     def test_not_enough_samples(self):
         assert_raises(ValueError, stats.anderson_ksamp, np.ones(5))
@@ -384,19 +447,12 @@ def test_result_attributes(self):
         t4 = np.array([34.0, 34.8, 34.8, 35.4, 37.2, 37.8, 41.2, 42.8])
 
         with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
+            sup.filter(UserWarning, message='p-value floored')
             res = stats.anderson_ksamp((t1, t2, t3, t4), midrank=False)
 
         attributes = ('statistic', 'critical_values', 'significance_level')
         check_named_results(res, attributes)
 
-    def test_overflow(self):
-        # when significance_level approximation overflows, should still return
-        with suppress_warnings() as sup:
-            sup.filter(UserWarning, message='approximate p-value')
-            res = stats.anderson_ksamp([[-20, -10] * 100, [-10, 40, 12] * 100])
-            assert_almost_equal(res[0], 272.796, 3)
-
 
 class TestAnsari(object):