Repository: statsmodels/statsmodels

Issue:
BUG: bug in outlier_test index if order is True
checking code for #2727
Looking at the code, I think there is a bug or order=False and the original pandas index is available. AFAICS the data is sorted but not the pandas index.

needs an example to check, because I'm not sure about the pandas behavior here.

most likely a bug in the returned index if `order` is trueish


Discussion:
-BUG confirmed, we get the wrong labels in the ordered array
-another problem the OLSResults method doesn't use all keywords of the outlier_test function, e.g. order is missing.
-labels keyword in function does not have a docstring entry

this works correctly:
`res.outlier_test().sort_values(['student_resid'], ascending=False)`

the following returns an incorrect index that has not been sorted to correspond to the sorted values:
```
import statsmodels.stats.outliers_influence as oi
ot2 = oi.outlier_test(res, order=True)
```

code:
```
    if labels is None:
        labels = getattr(model_results.model.data, 'row_labels', None)
```
is too late, the labels (if user provided) are sorted earlier, but not if the labels are taken from model.data

so this works also correctly and produces the same result as sorting the results dataframe
`oi.outlier_test(res, order=True, labels=res.model.data.row_labels)`

this means we can just move the lines for accessing model row_labels to the top

unit tests: there is only one test based on file search

stats.tests.test.test_diagnostic, test_outlier_test numbers against R
but the test uses explicitly provided labels
`res = oi.outlier_test(ndarray_mod, method='b', labels=labels, order=True)`

PR coming up later today

Resolving patch:
diff --git a/statsmodels/regression/linear_model.py b/statsmodels/regression/linear_model.py
--- a/statsmodels/regression/linear_model.py
+++ b/statsmodels/regression/linear_model.py
@@ -2556,7 +2556,8 @@ def get_influence(self):
         from statsmodels.stats.outliers_influence import OLSInfluence
         return OLSInfluence(self)
 
-    def outlier_test(self, method='bonf', alpha=.05):
+    def outlier_test(self, method='bonf', alpha=.05, labels=None,
+                 order=False, cutoff=None):
         """
         Test observations for outliers according to method
 
@@ -2576,6 +2577,16 @@ def outlier_test(self, method='bonf', alpha=.05):
             See `statsmodels.stats.multitest.multipletests` for details.
         alpha : float
             familywise error rate
+        labels : None or array_like
+            If `labels` is not None, then it will be used as index to the
+            returned pandas DataFrame. See also Returns below
+        order : bool
+            Whether or not to order the results by the absolute value of the
+            studentized residuals. If labels are provided they will also be sorted.
+        cutoff : None or float in [0, 1]
+            If cutoff is not None, then the return only includes observations with
+            multiple testing corrected p-values strictly below the cutoff. The
+            returned array or dataframe can be empty if t
 
         Returns
         -------
@@ -2591,7 +2602,8 @@ def outlier_test(self, method='bonf', alpha=.05):
         df = df_resid - 1.
         """
         from statsmodels.stats.outliers_influence import outlier_test
-        return outlier_test(self, method, alpha)
+        return outlier_test(self, method, alpha, labels=labels,
+                            order=order, cutoff=cutoff)
 
     def el_test(self, b0_vals, param_nums, return_weights=0,
                 ret_params=0, method='nm',
diff --git a/statsmodels/stats/outliers_influence.py b/statsmodels/stats/outliers_influence.py
--- a/statsmodels/stats/outliers_influence.py
+++ b/statsmodels/stats/outliers_influence.py
@@ -18,7 +18,7 @@
 # outliers test convenience wrapper
 
 def outlier_test(model_results, method='bonf', alpha=.05, labels=None,
-                 order=False):
+                 order=False, cutoff=None):
     """
     Outlier Tests for RegressionResults instances.
 
@@ -38,9 +38,17 @@ def outlier_test(model_results, method='bonf', alpha=.05, labels=None,
         See `statsmodels.stats.multitest.multipletests` for details.
     alpha : float
         familywise error rate
+    labels : None or array_like
+        If `labels` is not None, then it will be used as index to the
+        returned pandas DataFrame. See also Returns below
     order : bool
         Whether or not to order the results by the absolute value of the
         studentized residuals. If labels are provided they will also be sorted.
+    cutoff : None or float in [0, 1]
+        If cutoff is not None, then the return only includes observations with
+        multiple testing corrected p-values strictly below the cutoff. The
+        returned array or dataframe can be empty if there are no outlier
+        candidates at the specified cutoff.
 
     Returns
     -------
@@ -56,6 +64,8 @@ def outlier_test(model_results, method='bonf', alpha=.05, labels=None,
     df = df_resid - 1.
     """
     from scipy import stats # lazy import
+    if labels is None:
+        labels = getattr(model_results.model.data, 'row_labels', None)
     infl = getattr(model_results, 'get_influence', None)
     if infl is None:
         results = maybe_unwrap_results(model_results)
@@ -66,19 +76,23 @@ def outlier_test(model_results, method='bonf', alpha=.05, labels=None,
         idx = np.abs(resid).argsort()[::-1]
         resid = resid[idx]
         if labels is not None:
-            labels = np.array(labels)[idx].tolist()
+            labels = np.asarray(labels)[idx]
     df = model_results.df_resid - 1
     unadj_p = stats.t.sf(np.abs(resid), df) * 2
     adj_p = multipletests(unadj_p, alpha=alpha, method=method)
 
     data = np.c_[resid, unadj_p, adj_p[1]]
-    if labels is None:
-        labels = getattr(model_results.model.data, 'row_labels', None)
+    if cutoff is not None:
+        mask = data[:, -1] < cutoff
+        data = data[mask]
+    else:
+        mask = slice(None)
+
     if labels is not None:
         from pandas import DataFrame
         return DataFrame(data,
                          columns=['student_resid', 'unadj_p', method+"(p)"],
-                         index=labels)
+                         index=np.asarray(labels)[mask])
     return data
 
 #influence measures


Test patch:
diff --git a/statsmodels/stats/tests/test_diagnostic.py b/statsmodels/stats/tests/test_diagnostic.py
--- a/statsmodels/stats/tests/test_diagnostic.py
+++ b/statsmodels/stats/tests/test_diagnostic.py
@@ -16,9 +16,15 @@
 import os
 
 import numpy as np
+import pandas as pd
+
+# skipping some parts
+from distutils.version import LooseVersion
+PD_GE_17 = LooseVersion(pd.__version__) >= '0.17'
 
 from numpy.testing import (assert_, assert_almost_equal, assert_equal,
-                           assert_approx_equal, assert_allclose)
+                           assert_approx_equal, assert_allclose,
+                           assert_array_equal)
 import pytest
 
 from statsmodels.regression.linear_model import OLS, GLSAR
@@ -326,7 +332,7 @@ def test_acorr_ljung_box(self):
         res = self.res
 
         #general test
-        
+
         #> bt = Box.test(residuals(fm), lag=4, type = "Ljung-Box")
         #> mkhtest(bt, "ljung_box_4", "chi2")
         ljung_box_4 = dict(statistic=5.23587172795227, pvalue=0.263940335284713,
@@ -365,7 +371,7 @@ def test_acorr_ljung_box_big_default(self):
     def test_acorr_ljung_box_small_default(self):
         res = self.res
         #test with small dataset and default lag
-        
+
         #> bt = Box.test(residuals(fm), type = "Ljung-Box")
         #> mkhtest(bt, "ljung_box_small", "chi2")
         ljung_box_small = dict(statistic=9.61503968281915, pvalue=0.72507000996945,
@@ -920,6 +926,32 @@ def test_outlier_test():
     np.testing.assert_almost_equal(res.values, res2, 7)
     np.testing.assert_equal(res.index.tolist(), sorted_labels)  # pylint: disable-msg=E1103
 
+    data = pd.DataFrame(np.column_stack((endog, exog)),
+                        columns='y const var1 var2'.split(),
+                        index=labels)
+
+    # check `order` with pandas bug in #3971
+    res_pd = OLS.from_formula('y ~ const + var1 + var2 - 0', data).fit()
+
+    res_outl2 = oi.outlier_test(res_pd, method='b', order=True)
+    assert_almost_equal(res_outl2.values, res2, 7)
+    assert_equal(res_outl2.index.tolist(), sorted_labels)
+
+    if PD_GE_17:
+        # pandas < 0.17 does not have sort_values method
+        res_outl1 = res_pd.outlier_test(method='b')
+        res_outl1 = res_outl1.sort_values(['unadj_p'], ascending=True)
+        assert_almost_equal(res_outl1.values, res2, 7)
+        assert_equal(res_outl1.index.tolist(), sorted_labels)
+        assert_array_equal(res_outl2.index, res_outl1.index)
+
+
+    # additional keywords in method
+    res_outl3 = res_pd.outlier_test(method='b', order=True)
+    assert_equal(res_outl3.index.tolist(), sorted_labels)
+    res_outl4 = res_pd.outlier_test(method='b', order=True, cutoff=0.15)
+    assert_equal(res_outl4.index.tolist(), sorted_labels[:1])
+
 
 if __name__ == '__main__':
     import pytest