Repository: statsmodels/statsmodels

Issue:
ENH/REF isfinite check when using _MinimalWLS
GLM and RLM use _MinimalWLS starting in 0.9, merged in #3048

this does not include a isfinite check, and might hang or segfault with some linalg libraries if inf or nans are encountered.

E.g. 
GLM binomial (in original `distributed` notebook) used endog that are outside of the support of the distribution. This raised an exception for the isfinite check with WLS (added in 0.9 in the has-constant check)
AFAIR, in my messy testing for 0.9 release I ended up with cases like this that now have no isfinite protection before going to linalg.

My guess
- add check for support of endog, I think only binary model in `discrete` have them.
- If endog and exog are ok from the data, then we only need to check that `weights` are `finite` during IRLS iterations in GLM and RLM.



Discussion:
maybe worth looking at for final 0.9 release, but we don't have general policy for isfinite checks yet.

Resolving patch:
diff --git a/statsmodels/genmod/generalized_linear_model.py b/statsmodels/genmod/generalized_linear_model.py
--- a/statsmodels/genmod/generalized_linear_model.py
+++ b/statsmodels/genmod/generalized_linear_model.py
@@ -1203,10 +1203,10 @@ def _fit_irls(self, start_params=None, maxiter=100, tol=1e-8,
                             self.family.weights(mu))
             wlsendog = (lin_pred + self.family.link.deriv(mu) * (self.endog-mu)
                         - self._offset_exposure)
-            wls_results = reg_tools._MinimalWLS(
-                    wlsendog,
-                    wlsexog,
-                    self.weights).fit(method=wls_method)
+            wls_mod = reg_tools._MinimalWLS(wlsendog, wlsexog,
+                                            self.weights, check_endog=True,
+                                            check_weights=True)
+            wls_results = wls_mod.fit(method=wls_method)
             lin_pred = np.dot(self.exog, wls_results.params)
             lin_pred += self._offset_exposure
             mu = self.family.fitted(lin_pred)
diff --git a/statsmodels/regression/_tools.py b/statsmodels/regression/_tools.py
--- a/statsmodels/regression/_tools.py
+++ b/statsmodels/regression/_tools.py
@@ -1,9 +1,6 @@
-from collections import namedtuple
 import numpy as np
 from statsmodels.tools.tools import Bunch
 
-_MinimalWLSModel = namedtuple('_MinimalWLSModel', ['weights'])
-
 
 class _MinimalWLS(object):
     """
@@ -22,25 +19,37 @@ class _MinimalWLS(object):
         1d array of weights.  If you supply 1/W then the variables are pre-
         multiplied by 1/sqrt(W).  If no weights are supplied the default value
         is 1 and WLS reults are the same as OLS.
+    check_endog : bool, optional
+        Flag indicating whether to check for inf/nan in endog.
+        If True and any are found, ValueError is raised.
+    check_weights : bool, optional
+        Flag indicating whether to check for inf/nan in weights.
+        If True and any are found, ValueError is raised.
 
     Notes
     -----
-    Need resid, scale, fittedvalues, model.weights!
-        history['scale'].append(tmp_results.scale)
-        if conv == 'dev':
-            history['deviance'].append(self.deviance(tmp_results))
-        elif conv == 'sresid':
-            history['sresid'].append(tmp_results.resid/tmp_results.scale)
-        elif conv == 'weights':
-            history['weights'].append(tmp_results.model.weights)
-    Does not perform and checks on the input data
+    Provides only resid, scale, fittedvalues, model.weights which are used by
+    methods that iteratively apply WLS.
+
+    Does not perform any checks on the input data for type or shape
+    compatibility
     """
 
-    def __init__(self, endog, exog, weights=1.0):
+    msg = 'NaN, inf or invalid value detected in {0}, estimation infeasible.'
+
+    def __init__(self, endog, exog, weights=1.0, check_endog=False,
+                 check_weights=False):
         self.endog = endog
         self.exog = exog
         self.weights = weights
         w_half = np.sqrt(weights)
+        if check_weights:
+            if not np.all(np.isfinite(w_half)):
+                raise ValueError(self.msg.format('weights'))
+
+        if check_endog:
+            if not np.all(np.isfinite(endog)):
+                raise ValueError(self.msg.format('endog'))
 
         self.wendog = w_half * endog
         if np.isscalar(weights):
diff --git a/statsmodels/robust/robust_linear_model.py b/statsmodels/robust/robust_linear_model.py
--- a/statsmodels/robust/robust_linear_model.py
+++ b/statsmodels/robust/robust_linear_model.py
@@ -276,7 +276,8 @@ def fit(self, maxiter=50, tol=1e-8, scale_est='mad', init=None, cov='H1',
         while not converged:
             self.weights = self.M.weights(wls_results.resid/self.scale)
             wls_results = reg_tools._MinimalWLS(self.endog, self.exog,
-                                                weights=self.weights).fit()
+                                                weights=self.weights,
+                                                check_weights=True).fit()
             if update_scale is True:
                 self.scale = self._estimate_scale(wls_results.resid)
             history = self._update_history(wls_results, history, conv)


Test patch:
diff --git a/statsmodels/regression/tests/test_tools.py b/statsmodels/regression/tests/test_tools.py
--- a/statsmodels/regression/tests/test_tools.py
+++ b/statsmodels/regression/tests/test_tools.py
@@ -1,39 +1,62 @@
-from unittest import TestCase
-
 import numpy as np
 from numpy.testing import assert_allclose
+import pytest
 
 from statsmodels.regression.linear_model import WLS
 from statsmodels.regression._tools import _MinimalWLS
 
-class TestMinimalWLS(TestCase):
+
+class TestMinimalWLS(object):
     @classmethod
     def setup_class(cls):
         rs = np.random.RandomState(1234)
-        cls.exog1 = rs.randn(200,5)
+        cls.exog1 = rs.randn(200, 5)
         cls.endog1 = cls.exog1.sum(1) + rs.randn(200)
-        cls.weights1 = 1.0 + np.sin(np.arange(200.0)/100.0*np.pi)
-        cls.exog2 = rs.randn(50,1)
+        cls.weights1 = 1.0 + np.sin(np.arange(200.0) / 100.0 * np.pi)
+        cls.exog2 = rs.randn(50, 1)
         cls.endog2 = 0.3 * cls.exog2.ravel() + rs.randn(50)
-        cls.weights2 = 1.0 + np.log(np.arange(1.0,51.0))
+        cls.weights2 = 1.0 + np.log(np.arange(1.0, 51.0))
 
-    def test_equivalence_with_wls(self):
+    @pytest.mark.parametrize('check', [True, False])
+    def test_equivalence_with_wls(self, check):
         res = WLS(self.endog1, self.exog1).fit()
-        minres = _MinimalWLS(self.endog1, self.exog1).fit()
+        minres = _MinimalWLS(self.endog1, self.exog1,
+                             check_endog=check, check_weights=check).fit()
         assert_allclose(res.params, minres.params)
         assert_allclose(res.resid, minres.resid)
 
         res = WLS(self.endog2, self.exog2).fit()
-        minres = _MinimalWLS(self.endog2, self.exog2).fit()
+        minres = _MinimalWLS(self.endog2, self.exog2,
+                             check_endog=check, check_weights=check).fit()
         assert_allclose(res.params, minres.params)
         assert_allclose(res.resid, minres.resid)
 
         res = WLS(self.endog1, self.exog1, weights=self.weights1).fit()
-        minres = _MinimalWLS(self.endog1, self.exog1, weights=self.weights1).fit()
+        minres = _MinimalWLS(self.endog1, self.exog1, weights=self.weights1,
+                             check_endog=check, check_weights=check).fit()
         assert_allclose(res.params, minres.params)
         assert_allclose(res.resid, minres.resid)
 
         res = WLS(self.endog2, self.exog2, weights=self.weights2).fit()
-        minres = _MinimalWLS(self.endog2, self.exog2, weights=self.weights2).fit()
+        minres = _MinimalWLS(self.endog2, self.exog2, weights=self.weights2,
+                             check_endog=check, check_weights=check).fit()
         assert_allclose(res.params, minres.params)
         assert_allclose(res.resid, minres.resid)
+
+    @pytest.mark.parametrize('bad_value', [np.nan, np.inf])
+    def test_inf_nan(self, bad_value):
+        with pytest.raises(ValueError) as err:
+            endog = self.endog1.copy()
+            endog[0] = bad_value
+            _MinimalWLS(endog, self.exog1, self.weights1,
+                        check_endog=True, check_weights=True).fit()
+        assert err.type is ValueError
+        assert 'endog' in str(err)
+
+        with pytest.raises(ValueError) as err:
+            weights = self.weights1.copy()
+            weights[-1] = bad_value
+            _MinimalWLS(self.endog1, self.exog1, weights,
+                        check_endog=True, check_weights=True).fit()
+        assert err.type is ValueError
+        assert 'weights' in str(err)