internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: statsmodels/statsmodels

Issue:
REF: warnings rename ColinearityWarning to CollinearityWarning
https://groups.google.com/d/msg/pystatsmodels/S9VspZLn7No/tBN3AMY7BwAJ

for consistent spelling we should change it in tools.sm_exceptions



Discussion:

Resolving patch:
diff --git a/statsmodels/tools/sm_exceptions.py b/statsmodels/tools/sm_exceptions.py
--- a/statsmodels/tools/sm_exceptions.py
+++ b/statsmodels/tools/sm_exceptions.py
@@ -122,7 +122,7 @@ class HessianInversionWarning(ModelWarning):
     pass
 
 
-class ColinearityWarning(ModelWarning):
+class CollinearityWarning(ModelWarning):
     pass
 
 warnings.simplefilter('always', category=ModelWarning)


Test patch:
diff --git a/statsmodels/tsa/tests/test_stattools.py b/statsmodels/tsa/tests/test_stattools.py
--- a/statsmodels/tsa/tests/test_stattools.py
+++ b/statsmodels/tsa/tests/test_stattools.py
@@ -4,19 +4,19 @@
 import numpy as np
 import pandas as pd
 import pytest
-from numpy.testing import (assert_almost_equal, assert_equal, assert_raises, assert_,
-                           assert_allclose)
+from numpy.testing import (assert_almost_equal, assert_equal, assert_raises,
+                           assert_, assert_allclose)
 from pandas import Series, DatetimeIndex, DataFrame
 
 from statsmodels.compat.numpy import recarray_select
 from statsmodels.compat.python import lrange
 from statsmodels.datasets import macrodata, sunspots
-from statsmodels.tools.sm_exceptions import ColinearityWarning
-from statsmodels.tools.sm_exceptions import MissingDataError
+from statsmodels.tools.sm_exceptions import (CollinearityWarning,
+                                             MissingDataError)
 from statsmodels.tsa.stattools import (adfuller, acf, pacf_yw,
                                        pacf, grangercausalitytests,
-                                       coint, acovf, kpss, arma_order_select_ic,
-                                       levinson_durbin)
+                                       coint, acovf, kpss,
+                                       arma_order_select_ic, levinson_durbin)
 
 DECIMAL_8 = 8
 DECIMAL_6 = 6
@@ -364,7 +364,7 @@ def test_coint_identical_series():
     scale_e = 1
     np.random.seed(123)
     y = scale_e * np.random.randn(nobs)
-    warnings.simplefilter('always', ColinearityWarning)
+    warnings.simplefilter('always', CollinearityWarning)
     with warnings.catch_warnings(record=True) as w:
         c = coint(y, y, trend="c", maxlag=0, autolag=None)
     assert_equal(len(w), 1)
@@ -379,7 +379,7 @@ def test_coint_perfect_collinearity():
     np.random.seed(123)
     x = scale_e * np.random.randn(nobs, 2)
     y = 1 + x.sum(axis=1) + 1e-7 * np.random.randn(nobs)
-    warnings.simplefilter('always', ColinearityWarning)
+    warnings.simplefilter('always', CollinearityWarning)
     with warnings.catch_warnings(record=True) as w:
         c = coint(y, x, trend="c", maxlag=0, autolag=None)
     assert_equal(c[1], 0.0)
Repository: statsmodels/statsmodels

Issue:
[MAINT/CLN] remove function explicitly marked as duplicate
In the function docstring:
`duplicate: Skipper added sm.tools.drop_missing`

<b>update</b> The relevant function is not used outside this module; nor is the other function in this module.


Discussion:
you can delete the entire file wrapper.py
It's an experiment that never got any support
I don't like dispatch classes because they are too confusing and nobody asked for it in 8 years or so, AFAIR. 
(AFAIK the only one with ambiguous classes is ARMA/ARIMA)

i.e. we are not putting "everything but the kitchen sink" in a function like R's `lm`.


Resolving patch:
diff --git a/statsmodels/tools/wrappers.py b/statsmodels/tools/wrappers.py
deleted file mode 100644
--- a/statsmodels/tools/wrappers.py
+++ /dev/null
@@ -1,52 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Convenience Wrappers
-
-Created on Sat Oct 30 14:56:35 2010
-
-Author: josef-pktd
-License: BSD
-"""
-
-import numpy as np
-import statsmodels.api as sm
-from statsmodels import GLS, WLS, OLS
-
-def remove_nanrows(y, x):
-    '''remove common rows in [y,x] that contain at least one nan
-
-    TODO: this should be made more flexible,
-     arbitrary number of arrays and 1d or 2d arrays
-
-    duplicate: Skipper added sm.tools.drop_missing
-
-    '''
-    mask = ~np.isnan(y)
-    mask *= ~(np.isnan(x).any(-1))  #* or &
-    y = y[mask]
-    x = x[mask]
-    return y, x
-
-
-def linmod(y, x, weights=None, sigma=None, add_const=True, filter_missing=True,
-           **kwds):
-    '''get linear model with extra options for entry
-
-    dispatches to regular model class and does not wrap the output
-
-    If several options are exclusive, for example sigma and weights, then the
-    chosen class depends on the implementation sequence.
-    '''
-
-    if filter_missing:
-        y, x = remove_nanrows(y, x)
-        #do the same for masked arrays
-
-    if add_const:
-        x = sm.add_constant(x, prepend=True)
-
-    if not sigma is None:
-        return GLS(y, x, sigma=sigma, **kwds)
-    elif not weights is None:
-        return WLS(y, x, weights=weights, **kwds)
-    else:
-        return OLS(y, x, **kwds)


Test patch:
Repository: statsmodels/statsmodels

Issue:
Minor documentation bug in new state space model tools.
Throughout the various new state space models classes and methods, the variable `k_posdef` is described as:

The dimension of a guaranteed positive definite covariance matrix describing the shocks in the **measurement equation**,

but it seems that this is actually rank of the shocks that evolve the transition / state equations.

In [47]: sm.version.full_version
Out[47]: '0.8.0'



Discussion:
You're absolutely right - thanks for letting us know!
@ChadFulton is this still an issue?

Resolving patch:
diff --git a/statsmodels/tsa/statespace/kalman_filter.py b/statsmodels/tsa/statespace/kalman_filter.py
--- a/statsmodels/tsa/statespace/kalman_filter.py
+++ b/statsmodels/tsa/statespace/kalman_filter.py
@@ -66,7 +66,7 @@ class KalmanFilter(Representation):
         The dimension of the unobserved state process.
     k_posdef : int, optional
         The dimension of a guaranteed positive definite covariance matrix
-        describing the shocks in the measurement equation. Must be less than
+        describing the shocks in the transition equation. Must be less than
         or equal to `k_states`. Default is `k_states`.
     loglikelihood_burn : int, optional
         The number of initial periods during which the loglikelihood is not


Test patch:
Repository: statsmodels/statsmodels

Issue:
GenericLikelihoodModel.initialize sets self.score to approx_fprime
... I don't think that's going to work.


Discussion:
Also, sure would be nice if `fixed_paramsmask` was set in a predictable way.
needs review

E.g. I haven't looked at`fixed_paramsmask` in many years
AFAIK, it didn't get implemented much outside of GenericLikelihoodModels, but it has a unit tested example use case
https://github.com/statsmodels/statsmodels/blob/master/statsmodels/miscmodels/tmodel.py#L59

(equality constraints for nonlinear optimization are another medium/long term wishlist item. fit_constrained only allows constraints on the parameters associated with exog.)

GenericLikelihoodModel and the accompanying Results subclass are a mess.  It would be really nice to get it into better shape because

a) I would sleep better at night knowing that base.model was uniformly in good condition
b) you mentioned there being an API for Generic, and it would be great if that were made consistent and usable.
> base.model was uniformly in good condition

GenericLikelihoodModel is intentionally the "dirty" cousin of LikelihoodModel. It is supposed to have a default for everything except loglike, while the cleaner LikelihoodModel assumes that the subclasses provide all the missing pieces.

This does not mean that it has to be very dirty, and it could get another **round**.
Another issue that is open is that some of the GenericLM things can be moved to LM.

(e.g. I also just found that df_resid, df_model defautls don't work in a simple scipy.stats.distribution wrapper, on mailing list.)

Resolving patch:
diff --git a/statsmodels/base/model.py b/statsmodels/base/model.py
--- a/statsmodels/base/model.py
+++ b/statsmodels/base/model.py
@@ -761,7 +761,7 @@ def _set_extra_params_names(self, extra_params_names):
     # this is redundant and not used when subclassing
     def initialize(self):
         if not self.score:  # right now score is not optional
-            self.score = approx_fprime
+            self.score = lambda x: approx_fprime(x, self.loglike)
             if not self.hessian:
                 pass
         else:   # can use approx_hess_p if we have a gradient


Test patch:
Repository: statsmodels/statsmodels

Issue:
web._generate_url returns ValueError--> intentional?
```
            return ValueError('Input not understood')
```
seems like it should be `raise` instead of `return`.  Am I missing something?


Discussion:

Resolving patch:
diff --git a/statsmodels/tools/web.py b/statsmodels/tools/web.py
--- a/statsmodels/tools/web.py
+++ b/statsmodels/tools/web.py
@@ -4,7 +4,7 @@
 """
 import webbrowser
 
-from statsmodels.compat.python import urlencode
+from statsmodels.compat.python import urlencode, string_types
 from statsmodels import __version__
 
 BASE_URL = 'https://www.statsmodels.org/'
@@ -12,8 +12,8 @@
 
 def _generate_url(arg, stable):
     """
-    Parse inputs and return a correctly formatted URL or an error if the input
-    is not understandable
+    Parse inputs and return a correctly formatted URL or raises ValueError
+    if the input is not understandable
     """
     url = BASE_URL
     if stable:
@@ -23,7 +23,7 @@ def _generate_url(arg, stable):
 
     if arg is None:
         return url
-    elif type(arg) is str:
+    elif isinstance(arg, string_types):
         url += 'search.html?'
         url += urlencode({'q': arg})
         url += '&check_keywords=yes&area=default'
@@ -33,11 +33,11 @@ def _generate_url(arg, stable):
             func_name = func.__name__
             func_module = func.__module__
             if not func_module.startswith('statsmodels.'):
-                return ValueError('Function must be from statsmodels')
+                raise ValueError('Function must be from statsmodels')
             url += 'generated/'
             url += func_module + '.' + func_name + '.html'
-        except:
-            return ValueError('Input not understood')
+        except AttributeError:
+            raise ValueError('Input not understood')
     return url
 
 
@@ -71,7 +71,5 @@ def webdoc(arg=None, stable=None):
     """
     stable = __version__ if 'dev' not in __version__ else stable
     url_or_error = _generate_url(arg, stable)
-    if isinstance(url_or_error, ValueError):
-        raise url_or_error
     webbrowser.open(url_or_error)
     return None


Test patch:
Repository: statsmodels/statsmodels

Issue:
mlemodel.PredictionResults.summary_frame attached self.table... why?
@ChadFulton doesn't appear used, isn't a sm convention, and makes me wince because it isn't in `__init__`.  What's the reasoning for this?


Discussion:
No idea - looks like it came in the initial scikits commit.

(It often comes up first in Sublime Text when I'm actually trying to go to statespace/mlemodel, so I'm all for getting rid of it if it's not used :)
Where is this?
I remember we did this in the past to have access to the underlying dataframe (similar to a cached attribute), but I didn't find it anymore when I went briefly through some summary methods.
I found it. I used it in both genmod._prediction and in regression._prediction
I don't see a reason given in my original PR.
There is no caching, no use of cached attributes in PredictResults. So, I guess I wanted to cache the results at least somewhere to avoid recomputing.

design for get_prediction was preliminary (i.e. without thinking to hard about all the details), and never got a "finalizing" round. 




Resolving patch:
diff --git a/statsmodels/tsa/statespace/mlemodel.py b/statsmodels/tsa/statespace/mlemodel.py
--- a/statsmodels/tsa/statespace/mlemodel.py
+++ b/statsmodels/tsa/statespace/mlemodel.py
@@ -2921,7 +2921,6 @@ def summary_frame(self, endog=0, what='all', alpha=0.05):
         to_include['mean_ci_lower'] = ci_mean[:, endog]
         to_include['mean_ci_upper'] = ci_mean[:, k_endog + endog]
 
-        self.table = to_include
         # OrderedDict doesn't work to preserve sequence
         # pandas dict doesn't handle 2d_array
         # data = np.column_stack(list(to_include.values()))


Test patch:
Repository: statsmodels/statsmodels

Issue:
DOC: Matplotlib 3.0.0 seems to no longer have "matplotlib.sphinxext.only_directives"
Docbuild is failing. 

Seems we used to reference this in the sphinx build:

https://github.com/matplotlib/matplotlib/blob/2e0eb748d3c1b808ed89e86e80a2d1565b4cd896/lib/matplotlib/sphinxext/only_directives.py

But that module is now gone in 3.0.0. I haven't read the reasons/matplotlibs suggested workaround yet.




Discussion:

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -35,7 +35,6 @@
               'sphinx.ext.autosummary',
               'sphinx.ext.inheritance_diagram',
               'matplotlib.sphinxext.plot_directive',
-              'matplotlib.sphinxext.only_directives',
               'IPython.sphinxext.ipython_console_highlighting',
               'IPython.sphinxext.ipython_directive',
               'github',  # for GitHub links,
@@ -327,4 +326,4 @@
 import scipy.stats as stats
 import matplotlib.pyplot as plt
 import pandas as pd
-"""
\ No newline at end of file
+"""


Test patch:
Repository: statsmodels/statsmodels

Issue:
TST: missing seed and test failure in qsturng
try except leaves a variable undefined

test error in https://travis-ci.org/statsmodels/statsmodels/jobs/444183621

```
statsmodels/stats/libqsturng/qsturng_.py:731: in _qsturng
    r0_sq = _interpolate_p(p, r, v0)**2
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
p = 0.902896898398093, r = 9, v = 1
    def _interpolate_p(p, r, v):
        """
        interpolates p based on the values in the A table for the
        scalar value of r and the scalar value of v
        """
    
        # interpolate p (v should be in table)
        # if .5 < p < .75 use linear interpolation in q
        # if p > .75 use quadratic interpolation in log(y + r/v)
        # by -1. / (1. + 1.5 * _phi((1. + p)/2.))
    
        # find the 3 closest v values
        p0, p1, p2 = _select_ps(p)
        try:
            y0 = _func(A[(p0, v)], p0, r, v) + 1.
        except:
            print(p,r,v)
        y1 = _func(A[(p1, v)], p1, r, v) + 1.
        y2 = _func(A[(p2, v)], p2, r, v) + 1.
    
>       y_log0 = math.log(y0 + float(r)/float(v))
E       UnboundLocalError: local variable 'y0' referenced before assignment
```


Discussion:

Resolving patch:
diff --git a/statsmodels/stats/libqsturng/qsturng_.py b/statsmodels/stats/libqsturng/qsturng_.py
--- a/statsmodels/stats/libqsturng/qsturng_.py
+++ b/statsmodels/stats/libqsturng/qsturng_.py
@@ -531,6 +531,7 @@ def _interpolate_p(p, r, v):
         y0 = _func(A[(p0, v)], p0, r, v) + 1.
     except:
         print(p,r,v)
+        raise
     y1 = _func(A[(p1, v)], p1, r, v) + 1.
     y2 = _func(A[(p2, v)], p2, r, v) + 1.
 


Test patch:
Repository: statsmodels/statsmodels

Issue:
DOC: Incomplete Binomial docstring on endog format
I know there are some active discussions about revising the GLM Binomial implementation (see #5282), but in the mean time I wanted to highlight a somewhat confusing note in the documentation:

https://github.com/statsmodels/statsmodels/blob/7a33ef0ba2900f8e1d27fbd052c7a28d5a626342/statsmodels/genmod/families/family.py#L799

The documentation explains that there are three ways to pass endogenous variables to the Binomial GLM, but does not elaborate what those three ways are.

I'm happy to write a few sentences elaborating and submit a pull request if that helps. I _believe_ the three ways are as follows:

* A vector of 0 or 1 values (indicating failure or success respectively).
* A vector of proportions, indicating the proportion of successes, with parameter `freq_weights` containing the number of trials for each row.
* A `N × 2` matrix, where values in the first column represent the success count and values in the second column represent the failure count.

Thanks!


Discussion:
This two are the ways that existed for a long time

- A vector of 0 or 1 values (indicating failure or success respectively).
- A N × 2 matrix, where values in the first column represent the success count and values in the second column represent the failure count.

There are equivalent ways now after the introduction of weights. There are some comparison of different ways of specifying the same basic models in notebooks that we made during the weights addition. 

- A vector of proportions, indicating the proportion of successes, with parameter freq_weights containing the number of trials for each row.
This needs to use `var_weights` and not `freq_weights`. This is the main version that R's GLM uses, AFAIR.

`var_weights` are used when endog is the mean of response values for observations that have the same exog (but not necessarily the same endog).
`freq_weights` are used when cases are replicated, i.e. combining observations that have the same exog AND the same endog.

Improvements to the documentation is welcome.




Resolving patch:
diff --git a/statsmodels/genmod/families/family.py b/statsmodels/genmod/families/family.py
--- a/statsmodels/genmod/families/family.py
+++ b/statsmodels/genmod/families/family.py
@@ -796,7 +796,15 @@ class Binomial(Family):
 
     Notes
     -----
-    endog for Binomial can be specified in one of three ways.
+    endog for Binomial can be specified in one of three ways:
+    A 1d array of 0 or 1 values, indicating failure or success 
+    respectively.
+    A 2d array, with two columns. The first column represents the 
+    success count and the second column represents the failure 
+    count.
+    A 1d array of proportions, indicating the proportion of 
+    successes, with parameter `var_weights` containing the 
+    number of trials for each row.
     """
 
     links = [L.logit, L.probit, L.cauchy, L.log, L.cloglog, L.identity]


Test patch:
Repository: statsmodels/statsmodels

Issue:
modify comments in stats.proportion.proportion_confint
methods in ['normal', 'agresti_coull', 'beta', 'wilson', 'binom_test'] are all available(in line 88~143), but the comments was not updated. This PR updates the comments. 


Discussion:

Resolving patch:
diff --git a/statsmodels/stats/proportion.py b/statsmodels/stats/proportion.py
--- a/statsmodels/stats/proportion.py
+++ b/statsmodels/stats/proportion.py
@@ -26,7 +26,8 @@ def proportion_confint(count, nobs, alpha=0.05, method='normal'):
         total number of trials
     alpha : float in (0, 1)
         significance level, default 0.05
-    method : string in ['normal']
+    method : {'normal', 'agresti_coull', 'beta', 'wilson', 'binom_test'}
+        default: 'normal'
         method to use for confidence interval,
         currently available methods :
 


Test patch:
Repository: statsmodels/statsmodels

Issue:
Fix MICEData SettingWithCopyWarning on mixed-type dataframes
I believe this should fix #5430; I have an automated test that passes, at least for integer dtypes.


Discussion:

Resolving patch:
diff --git a/statsmodels/imputation/mice.py b/statsmodels/imputation/mice.py
--- a/statsmodels/imputation/mice.py
+++ b/statsmodels/imputation/mice.py
@@ -395,7 +395,7 @@ def _store_changes(self, col, vals):
 
         ix = self.ix_miss[col]
         if len(ix) > 0:
-            self.data[col].iloc[ix] = np.atleast_1d(vals)
+            self.data.iloc[ix, self.data.columns.get_loc(col)] = np.atleast_1d(vals)
 
     def update_all(self, n_iter=1):
         """


Test patch:
diff --git a/statsmodels/imputation/tests/test_mice.py b/statsmodels/imputation/tests/test_mice.py
--- a/statsmodels/imputation/tests/test_mice.py
+++ b/statsmodels/imputation/tests/test_mice.py
@@ -4,6 +4,7 @@
 from statsmodels.imputation import mice
 import statsmodels.api as sm
 from numpy.testing import assert_equal, assert_allclose
+import warnings
 
 try:
     import matplotlib.pyplot as plt
@@ -106,6 +107,26 @@ def test_default(self):
         assert_equal(exog_obs.shape, [190, 6])
         assert_equal(exog_miss.shape, [10, 6])
 
+    def test_settingwithcopywarning(self):
+        "Test that MICEData does not throw a SettingWithCopyWarning when imputing (https://github.com/statsmodels/statsmodels/issues/5430)"
+
+        df = gendat()
+        # There need to be some ints in here for the error to be thrown
+        df['intcol'] = np.arange(len(df))
+        df['intcol'] = df.intcol.astype('int32')
+
+        miceData = mice.MICEData(df)
+
+        with pd.option_context('mode.chained_assignment', 'warn'):
+            with warnings.catch_warnings(record=True) as ws:
+                warnings.simplefilter('always')
+                miceData.update_all()
+
+                # on Python 3.4, throws warning
+                # "DeprecationWarning('pandas.core.common.is_categorical_dtype is deprecated. import from the public API:
+                # pandas.api.types.is_categorical_dtype instead',)"
+                # ignore this warning, as this is not what is being tested in this test
+                assert ((len(ws) == 0) or all([w.category == DeprecationWarning for w in ws]))
 
     def test_next_sample(self):