Repository: statsmodels/statsmodels

Issue:
Calculate adjusted p-values for Tukey HSD Test
1. calculate adjusted p-values when using the tukeyhsd function
2. add the p-vals to the results table and as an attribute of the TukeyHSDResults object


Discussion:

Resolving patch:
diff --git a/statsmodels/sandbox/stats/multicomp.py b/statsmodels/sandbox/stats/multicomp.py
--- a/statsmodels/sandbox/stats/multicomp.py
+++ b/statsmodels/sandbox/stats/multicomp.py
@@ -176,61 +176,88 @@ def get_tukeyQcrit2(k, df, alpha=0.05):
     return qsturng(1-alpha, k, df)
 
 
-def Tukeythreegene(first,second,third):
-    #Performing the Tukey HSD post-hoc test for three genes
-##   qwb = xlrd.open_workbook('F:/Lab/bioinformatics/qcrittable.xls')
-##    #opening the workbook containing the q crit table
-##   qwb.sheet_names()
-##   qcrittable = qwb.sheet_by_name(u'Sheet1')
-
-    firstmean = numpy.mean(first) #means of the three arrays
+def get_tukey_pvalue(k, df, q):
+    '''
+    return adjusted p-values for Tukey's HSD
+
+    Parameters
+    ----------
+    k : int in {2, ..., 10}
+        number of tests
+    df : int
+        degrees of freedom of error term
+    q : scalar, array_like; q >= 0
+        quantile value of Studentized Range
+
+    '''
+
+    from statsmodels.stats.libqsturng import psturng
+    return psturng(q, k, df)
+
+
+def Tukeythreegene(first, second, third):
+    # Performing the Tukey HSD post-hoc test for three genes
+    # qwb = xlrd.open_workbook('F:/Lab/bioinformatics/qcrittable.xls')
+    # #opening the workbook containing the q crit table
+    # qwb.sheet_names()
+    # qcrittable = qwb.sheet_by_name(u'Sheet1')
+
+    # means of the three arrays
+    firstmean = numpy.mean(first)
     secondmean = numpy.mean(second)
     thirdmean = numpy.mean(third)
 
-    firststd = numpy.std(first) #standard deviations of the threearrays
+    # standard deviations of the threearrays
+    firststd = numpy.std(first)
     secondstd = numpy.std(second)
     thirdstd = numpy.std(third)
 
-    firsts2 = math.pow(firststd,2) #standard deviation squared of the three arrays
-    seconds2 = math.pow(secondstd,2)
-    thirds2 = math.pow(thirdstd,2)
+    # standard deviation squared of the three arrays
+    firsts2 = math.pow(firststd, 2)
+    seconds2 = math.pow(secondstd, 2)
+    thirds2 = math.pow(thirdstd, 2)
 
-    mserrornum = firsts2*2+seconds2*2+thirds2*2 #numerator for mean square error
-    mserrorden = (len(first)+len(second)+len(third))-3 #denominator for mean square error
-    mserror = mserrornum/mserrorden #mean square error
+    # numerator for mean square error
+    mserrornum = firsts2 * 2 + seconds2 * 2 + thirds2 * 2
+    # denominator for mean square error
+    mserrorden = (len(first) + len(second) + len(third)) - 3
+    mserror = mserrornum / mserrorden  # mean square error
 
-    standarderror = math.sqrt(mserror/len(first))
-    #standard error, which is square root of mserror and the number of samples in a group
+    standarderror = math.sqrt(mserror / len(first))
+    # standard error, which is square root of mserror and
+    # the number of samples in a group
 
-    dftotal = len(first)+len(second)+len(third)-1 #various degrees of freedom
+    # various degrees of freedom
+    dftotal = len(first) + len(second) + len(third) - 1
     dfgroups = 2
-    dferror = dftotal-dfgroups
+    dferror = dftotal - dfgroups  # noqa: F841
 
-    qcrit = 0.5 # fix arbitrary#qcrittable.cell(dftotal, 3).value
+    qcrit = 0.5  # fix arbitrary#qcrittable.cell(dftotal, 3).value
     qcrit = get_tukeyQcrit(3, dftotal, alpha=0.05)
-    #getting the q critical value, for degrees of freedom total and 3 groups
+    # getting the q critical value, for degrees of freedom total and 3 groups
 
-    qtest3to1 = (math.fabs(thirdmean-firstmean))/standarderror
-    #calculating q test statistic values
-    qtest3to2 = (math.fabs(thirdmean-secondmean))/standarderror
-    qtest2to1 = (math.fabs(secondmean-firstmean))/standarderror
+    qtest3to1 = (math.fabs(thirdmean - firstmean)) / standarderror
+    # calculating q test statistic values
+    qtest3to2 = (math.fabs(thirdmean - secondmean)) / standarderror
+    qtest2to1 = (math.fabs(secondmean - firstmean)) / standarderror
 
     conclusion = []
 
-##    print(qcrit
+    # print(qcrit
     print(qtest3to1)
     print(qtest3to2)
     print(qtest2to1)
 
-    if(qtest3to1>qcrit): #testing all q test statistic values to q critical values
+    # testing all q test statistic values to q critical values
+    if qtest3to1 > qcrit:
         conclusion.append('3to1null')
     else:
         conclusion.append('3to1alt')
-    if(qtest3to2>qcrit):
+    if qtest3to2 > qcrit:
         conclusion.append('3to2null')
     else:
         conclusion.append('3to2alt')
-    if(qtest2to1>qcrit):
+    if qtest2to1 > qcrit:
         conclusion.append('2to1null')
     else:
         conclusion.append('2to1alt')
@@ -599,6 +626,7 @@ class TukeyHSDResults(object):
     std_pairs : standard deviation of pairwise mean differences
     q_crit : critical value of studentized range statistic at given alpha
     halfwidths : half widths of simultaneous confidence interval
+    pvalues : adjusted p-values from the HSD test
 
     Notes
     -----
@@ -610,7 +638,7 @@ class TukeyHSDResults(object):
     """
     def __init__(self, mc_object, results_table, q_crit, reject=None,
                  meandiffs=None, std_pairs=None, confint=None, df_total=None,
-                 reject2=None, variance=None):
+                 reject2=None, variance=None, pvalues=None):
 
         self._multicomp = mc_object
         self._results_table = results_table
@@ -622,9 +650,10 @@ def __init__(self, mc_object, results_table, q_crit, reject=None,
         self.df_total = df_total
         self.reject2 = reject2
         self.variance = variance
+        self.pvalues = pvalues
         # Taken out of _multicomp for ease of access for unknowledgeable users
         self.data = self._multicomp.data
-        self.groups =self._multicomp.groups
+        self.groups = self._multicomp.groups
         self.groupsunique = self._multicomp.groupsunique
 
     def __str__(self):
@@ -946,9 +975,9 @@ def allpairtest(self, testfunc, alpha=0.05, method='bonf', pvalidx=1):
         return results_table, (res, reject, pvals_corrected,
                                alphacSidak, alphacBonf), resarr
 
-
     def tukeyhsd(self, alpha=0.05):
-        """Tukey's range test to compare means of all pairs of groups
+        """
+        Tukey's range test to compare means of all pairs of groups
 
         Parameters
         ----------
@@ -962,34 +991,38 @@ def tukeyhsd(self, alpha=0.05):
             calculations
         """
         self.groupstats = GroupsStats(
-                            np.column_stack([self.data, self.groupintlab]),
-                            useranks=False)
+            np.column_stack([self.data, self.groupintlab]),
+            useranks=False)
 
         gmeans = self.groupstats.groupmean
-        gnobs = self.groupstats.groupnobs        #var_ = self.groupstats.groupvarwithin() #possibly an error in varcorrection in this case
+        gnobs = self.groupstats.groupnobs
+        # var_ = self.groupstats.groupvarwithin()
+        # #possibly an error in varcorrection in this case
         var_ = np.var(self.groupstats.groupdemean(), ddof=len(gmeans))
-        #res contains: 0:(idx1, idx2), 1:reject, 2:meandiffs, 3: std_pairs, 4:confint, 5:q_crit,
-        #6:df_total, 7:reject2
+        # res contains: 0:(idx1, idx2), 1:reject, 2:meandiffs, 3: std_pairs,
+        # 4:confint, 5:q_crit, 6:df_total, 7:reject2, 8: pvals
         res = tukeyhsd(gmeans, gnobs, var_, df=None, alpha=alpha, q_crit=None)
 
-        resarr = np.array(lzip(self.groupsunique[res[0][0]], self.groupsunique[res[0][1]],
-                                  np.round(res[2],4),
-                                  np.round(res[4][:, 0],4),
-                                  np.round(res[4][:, 1],4),
-                                  res[1]),
-                       dtype=[('group1', object),
-                              ('group2', object),
-                              ('meandiff',float),
-                              ('lower',float),
-                              ('upper',float),
-                              ('reject', np.bool8)])
+        resarr = np.array(lzip(self.groupsunique[res[0][0]],
+                               self.groupsunique[res[0][1]],
+                               np.round(res[2], 4),
+                               np.round(res[8], 4),
+                               np.round(res[4][:, 0], 4),
+                               np.round(res[4][:, 1], 4),
+                               res[1]),
+                          dtype=[('group1', object),
+                                 ('group2', object),
+                                 ('meandiff', float),
+                                 ('p-adj', float),
+                                 ('lower', float),
+                                 ('upper', float),
+                                 ('reject', np.bool8)])
         results_table = SimpleTable(resarr, headers=resarr.dtype.names)
-        results_table.title = 'Multiple Comparison of Means - Tukey HSD,' + \
+        results_table.title = 'Multiple Comparison of Means - Tukey HSD, ' + \
                               'FWER=%4.2f' % alpha
 
         return TukeyHSDResults(self, results_table, res[5], res[1], res[2],
-                               res[3], res[4], res[6], res[7], var_)
-
+                               res[3], res[4], res[6], res[7], var_, res[8])
 
 
 def rankdata(x):
@@ -1278,14 +1311,17 @@ def tukeyhsd(mean_all, nobs_all, var_all, df=None, alpha=0.05, q_crit=None):
     if q_crit is None:
         q_crit = get_tukeyQcrit2(n_means, df_total, alpha=alpha)
 
+    pvalues = get_tukey_pvalue(n_means, df_total, st_range)
+
     reject = st_range > q_crit
     crit_int = std_pairs * q_crit
     reject2 = np.abs(meandiffs) > crit_int
 
     confint = np.column_stack((meandiffs - crit_int, meandiffs + crit_int))
 
-    return (idx1, idx2), reject, meandiffs, std_pairs, confint, q_crit, \
-           df_total, reject2
+    return ((idx1, idx2), reject, meandiffs, std_pairs, confint, q_crit,
+            df_total, reject2, pvalues)
+
 
 def simultaneous_ci(q_crit, var, groupnobs, pairindices=None):
     """Compute simultaneous confidence intervals for comparison of means.
diff --git a/statsmodels/stats/multicomp.py b/statsmodels/stats/multicomp.py
--- a/statsmodels/stats/multicomp.py
+++ b/statsmodels/stats/multicomp.py
@@ -8,11 +8,12 @@
 from statsmodels.sandbox.stats.multicomp import (  # noqa:F401
     tukeyhsd, MultiComparison)
 
+__all__ = ['tukeyhsd', 'MultiComparison']
 
-def pairwise_tukeyhsd(endog, groups, alpha=0.05):
-    '''calculate all pairwise comparisons with TukeyHSD confidence intervals
 
-    this is just a wrapper around tukeyhsd method of MultiComparison
+def pairwise_tukeyhsd(endog, groups, alpha=0.05):
+    """
+    Calculate all pairwise comparisons with TukeyHSD confidence intervals
 
     Parameters
     ----------
@@ -27,14 +28,17 @@ def pairwise_tukeyhsd(endog, groups, alpha=0.05):
     -------
     results : TukeyHSDResults instance
         A results class containing relevant data and some post-hoc
-        calculations
+        calculations, including adjusted p-value
+
+    Notes
+    -----
+    This is just a wrapper around tukeyhsd method of MultiComparison
 
     See Also
     --------
     MultiComparison
     tukeyhsd
     statsmodels.sandbox.stats.multicomp.TukeyHSDResults
-
-    '''
+    """
 
     return MultiComparison(endog, groups).tukeyhsd(alpha=alpha)


Test patch:
diff --git a/statsmodels/stats/tests/test_multi.py b/statsmodels/stats/tests/test_multi.py
--- a/statsmodels/stats/tests/test_multi.py
+++ b/statsmodels/stats/tests/test_multi.py
@@ -353,7 +353,12 @@ def test_tukeyhsd():
     pairs, reject, meandiffs, std_pairs, confint, q_crit = myres[:6]
     assert_almost_equal(meandiffs, res[:, 0], decimal=5)
     assert_almost_equal(confint, res[:, 1:3], decimal=2)
-    assert_equal(reject, res[:, 3]<0.05)
+    assert_equal(reject, res[:, 3] < 0.05)
+
+    # check p-values (divergence of high values is expected)
+    small_pvals_idx = [2, 5, 7, 9]
+    assert_allclose(myres[8][small_pvals_idx], res[small_pvals_idx, 3],
+                    rtol=1e-3)
 
 
 def test_local_fdr():