Repository: statsmodels/statsmodels

Issue:
ENH: option to disable GLM Perfect Separation exception
I want to estimate a saturated model for loglinear analysis, but I get an PerfectSeparation error for Poisson.

It looks like GLM doesn't have an option to disable Perfect Separation exception an to turn it into a warning.



Discussion:
I ran into this again for a one-way poisson example with unique exog and exposure, 
However, exception is only raised in irls, using `method='bfgs'` avoids the exception

bump and adding bug label
related: discrete has an attribute option to disable the exception, but there is no option to warn instead of the exception, i.e. we need also a PerfectSeparationWarning

also instead of an attribute we should have a keyword option to choose what to do in the case of perfect separation.

aside: there should be another related issue about using the perfect separation callback in non-binary models like the count models.
bump
https://stackoverflow.com/questions/59902037/perfect-separation-on-linear-model

I will just convert this to a Warning.
The user can set Warning to "error", but I will default to just warn

problems with test in discrete if I switch to warning instead of error:

example has convergence failure with default optimizer (newton)
When I switch to "bfgs", then it converges, but the PerfectSeparationWarning is not issued.
However, with have an extra text in the summary (I didn't remember adding this). The warning is added in `BinaryResults.summary`
In this example all standard errors are huge, so it's clear that params are not (well) identified.
Also, even the old code does not raise PerfectSeparationError when method="bfgs". So the check is not loose enough to catch this case.

```
> ...\statsmodels\statsmodels\discrete\tests\test_discrete.py(1576)test_perfect_prediction()
-> res = mod.fit(method="bfgs", disp=False, maxiter=50)  # should not raise but should warn
(Pdb) p res.mle_retvals['converged']
True
(Pdb) res.resid.var()
*** AttributeError: 'LogitResults' object has no attribute 'resid'
(Pdb) res.resid_response.var()
1.1297028686877065e-10
(Pdb) res.resid_response.mean()
1.1228335390697837e-06
(Pdb) res.summary()
<class 'statsmodels.iolib.summary.Summary'>
"""
                           Logit Regression Results
==============================================================================
Dep. Variable:                      y   No. Observations:                  100
Model:                          Logit   Df Residuals:                       95
Method:                           MLE   Df Model:                            4
Date:                Fri, 02 Dec 2022   Pseudo R-squ.:                   1.000
Time:                        10:27:32   Log-Likelihood:            -0.00011486
converged:                       True   LL-Null:                       -69.315
Covariance Type:            nonrobust   LLR p-value:                 5.547e-29
==============================================================================
                 coef    std err          z      P>|z|      [0.025      0.975]
------------------------------------------------------------------------------
const         -1.5110   5157.716     -0.000      1.000   -1.01e+04    1.01e+04
x1            -2.4487   1725.063     -0.001      0.999   -3383.510    3378.612
x2            -8.4204   2017.738     -0.004      0.997   -3963.114    3946.273
x3            12.7030   1318.984      0.010      0.992   -2572.458    2597.864
x4             5.5326   1986.007      0.003      0.998   -3886.971    3898.036
==============================================================================

Possibly complete quasi-separation: A fraction 0.99 of observations can be
perfectly predicted. This might indicate that there is complete
quasi-separation. In this case some parameters will not be identified.
"""
```


Resolving patch:
diff --git a/statsmodels/discrete/discrete_model.py b/statsmodels/discrete/discrete_model.py
--- a/statsmodels/discrete/discrete_model.py
+++ b/statsmodels/discrete/discrete_model.py
@@ -42,8 +42,9 @@
 from statsmodels.tools.numdiff import approx_fprime_cs
 from statsmodels.tools.sm_exceptions import (
     PerfectSeparationError,
+    PerfectSeparationWarning,
     SpecificationWarning,
-)
+    )
 
 
 try:
@@ -180,7 +181,7 @@ class DiscreteModel(base.LikelihoodModel):
     def __init__(self, endog, exog, check_rank=True, **kwargs):
         self._check_rank = check_rank
         super().__init__(endog, exog, **kwargs)
-        self.raise_on_perfect_prediction = True
+        self.raise_on_perfect_prediction = False  # keep for backwards compat
         self.k_extra = 0
 
     def initialize(self):
@@ -213,10 +214,15 @@ def pdf(self, X):
     def _check_perfect_pred(self, params, *args):
         endog = self.endog
         fittedvalues = self.cdf(np.dot(self.exog, params[:self.exog.shape[1]]))
-        if (self.raise_on_perfect_prediction and
-                np.allclose(fittedvalues - endog, 0)):
-            msg = "Perfect separation detected, results not available"
-            raise PerfectSeparationError(msg)
+        if np.allclose(fittedvalues - endog, 0):
+            if self.raise_on_perfect_prediction:
+                # backwards compatibility for attr raise_on_perfect_prediction
+                msg = "Perfect separation detected, results not available"
+                raise PerfectSeparationError(msg)
+            else:
+                msg = ("Perfect separation or prediction detected, "
+                       "parameter may not be identified")
+                warnings.warn(msg, category=PerfectSeparationWarning)
 
     @Appender(base.LikelihoodModel.fit.__doc__)
     def fit(self, start_params=None, method='newton', maxiter=35,
@@ -5156,7 +5162,7 @@ def summary(self, yname=None, xname=None, title=None, alpha=.05,
         etext = []
         if predclose_sum == len(fittedvalues):  # TODO: nobs?
             wstr = "Complete Separation: The results show that there is"
-            wstr += "complete separation.\n"
+            wstr += "complete separation or perfect prediction.\n"
             wstr += "In this case the Maximum Likelihood Estimator does "
             wstr += "not exist and the parameters\n"
             wstr += "are not identified."
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -47,7 +47,7 @@
 from statsmodels.tools.sm_exceptions import (
     DomainWarning,
     HessianInversionWarning,
-    PerfectSeparationError,
+    PerfectSeparationWarning,
 )
 from statsmodels.tools.validation import float_like
 
@@ -1248,8 +1248,9 @@ def _fit_irls(self, start_params=None, maxiter=100, tol=1e-8,
             history = self._update_history(wls_results, mu, history)
             self.scale = self.estimate_scale(mu)
             if endog.squeeze().ndim == 1 and np.allclose(mu - endog, 0):
-                msg = "Perfect separation detected, results not available"
-                raise PerfectSeparationError(msg)
+                msg = ("Perfect separation or prediction detected, "
+                       "parameter may not be identified")
+                warnings.warn(msg, category=PerfectSeparationWarning)
             converged = _check_convergence(criterion, iteration + 1, atol,
                                            rtol)
             if converged:
diff --git a/statsmodels/tools/sm_exceptions.py b/statsmodels/tools/sm_exceptions.py
--- a/statsmodels/tools/sm_exceptions.py
+++ b/statsmodels/tools/sm_exceptions.py
@@ -231,6 +231,14 @@ class CollinearityWarning(ModelWarning):
     pass
 
 
+class PerfectSeparationWarning(ModelWarning):
+    """
+    Perfect separation or prediction
+    """
+
+    pass
+
+
 class InfeasibleTestError(RuntimeError):
     """
     Test statistic cannot be computed


Test patch:
diff --git a/statsmodels/discrete/tests/test_discrete.py b/statsmodels/discrete/tests/test_discrete.py
--- a/statsmodels/discrete/tests/test_discrete.py
+++ b/statsmodels/discrete/tests/test_discrete.py
@@ -1558,20 +1558,24 @@ def test_perfect_prediction():
     iris_dir = os.path.abspath(iris_dir)
     iris = np.genfromtxt(os.path.join(iris_dir, 'iris.csv'), delimiter=",",
                          skip_header=1)
-    y = iris[:,-1]
-    X = iris[:,:-1]
+    y = iris[:, -1]
+    X = iris[:, :-1]
     X = X[y != 2]
     y = y[y != 2]
     X = sm.add_constant(X, prepend=True)
-    mod = Logit(y,X)
+    mod = Logit(y, X)
+    mod.raise_on_perfect_prediction = True
     assert_raises(PerfectSeparationError, mod.fit, maxiter=1000)
-    #turn off raise PerfectSeparationError
+    # turn off raise PerfectSeparationError
     mod.raise_on_perfect_prediction = False
     # this will raise if you set maxiter high enough with a singular matrix
     with pytest.warns(ConvergenceWarning):
         res = mod.fit(disp=False, maxiter=50)  # should not raise but does warn
     assert_(not res.mle_retvals['converged'])
 
+    # The following does not warn but message in summary()
+    mod.fit(method="bfgs", disp=False, maxiter=50)
+
 
 def test_poisson_predict():
     #GH: 175, make sure poisson predict works without offset and exposure
diff --git a/statsmodels/genmod/tests/test_glm.py b/statsmodels/genmod/tests/test_glm.py
--- a/statsmodels/genmod/tests/test_glm.py
+++ b/statsmodels/genmod/tests/test_glm.py
@@ -30,7 +30,7 @@
 )
 from statsmodels.tools.sm_exceptions import (
     DomainWarning,
-    PerfectSeparationError,
+    PerfectSeparationWarning,
     ValueWarning,
 )
 from statsmodels.tools.tools import add_constant
@@ -1003,9 +1003,9 @@ def test_perfect_pred(iris):
     y = y[y != 2]
     X = add_constant(X, prepend=True)
     glm = GLM(y, X, family=sm.families.Binomial())
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", category=RuntimeWarning)
-        assert_raises(PerfectSeparationError, glm.fit)
+
+    with pytest.warns(PerfectSeparationWarning):
+        glm.fit()
 
 
 def test_score_test_ols():