Repository: statsmodels/statsmodels

Issue:
Backport Python 3.11 support to 0.13.x maintenance branch
_Work in progress._



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -53,13 +53,13 @@
     for line in req.readlines():
         INSTALL_REQUIRES.append(line.split("#")[0].strip())
 
+CYTHON_MIN_VER = "0.29.32"  # released 2020
+
 DEVELOP_REQUIRES = []
 with open("requirements-dev.txt", encoding="utf-8") as req:
     for line in req.readlines():
         DEVELOP_REQUIRES.append(line.split("#")[0].strip())
 
-CYTHON_MIN_VER = "0.29.32"  # released 2020
-
 EXTRAS_REQUIRE = {
     "build": ["cython>=" + CYTHON_MIN_VER],
     "develop": ["cython>=" + CYTHON_MIN_VER] + DEVELOP_REQUIRES,
diff --git a/statsmodels/regression/linear_model.py b/statsmodels/regression/linear_model.py
--- a/statsmodels/regression/linear_model.py
+++ b/statsmodels/regression/linear_model.py
@@ -33,7 +33,7 @@
 from __future__ import annotations
 
 from statsmodels.compat.pandas import Appender
-from statsmodels.compat.python import lrange, lzip
+from statsmodels.compat.python import Literal, lrange, lzip
 
 from typing import Sequence
 import warnings
@@ -46,14 +46,13 @@
 import statsmodels.base.wrapper as wrap
 from statsmodels.emplike.elregress import _ELRegOpts
 # need import in module instead of lazily to copy `__doc__`
+from statsmodels.regression import _prediction as pred
 from statsmodels.regression._prediction import PredictionResults
 from statsmodels.tools.decorators import cache_readonly, cache_writable
 from statsmodels.tools.sm_exceptions import InvalidTestWarning, ValueWarning
 from statsmodels.tools.tools import pinv_extended
 from statsmodels.tools.validation import bool_like, float_like, string_like
 
-from . import _prediction as pred
-
 __docformat__ = 'restructuredtext en'
 
 __all__ = ['GLS', 'WLS', 'OLS', 'GLSAR', 'PredictionResults',
@@ -250,8 +249,25 @@ def whiten(self, x):
         """
         raise NotImplementedError("Subclasses must implement.")
 
-    def fit(self, method="pinv", cov_type='nonrobust', cov_kwds=None,
-            use_t=None, **kwargs):
+    def fit(
+            self,
+            method: Literal["pinv", "qr"] = "pinv",
+            cov_type: Literal[
+                "nonrobust",
+                "fixed scale",
+                "HC0",
+                "HC1",
+                "HC2",
+                "HC3",
+                "HAC",
+                "hac-panel",
+                "hac-groupsum",
+                "cluster",
+            ] = "nonrobust",
+            cov_kwds=None,
+            use_t: bool | None = None,
+            **kwargs
+    ):
         """
         Full fit of the model.
 


Test patch:
diff --git a/docs/source/dev/test_notes.rst b/docs/source/dev/test_notes.rst
--- a/docs/source/dev/test_notes.rst
+++ b/docs/source/dev/test_notes.rst
@@ -7,11 +7,11 @@ Setting up development environment locally
 ------------------------------------------
 Follow our :ref:`installation instructions <install>` and set up a suitable
 environment to build statsmodels from source. We recommend that you develop
-using a development install of statsmodels by running:
+using a development install of statsmodels in a `venv` by running:
 
 .. code-block:: bash
-
-   pip install -e .
+    python -m venv .venv
+    python -m pip install -e ".[develop]"
 
 from the root directory of the git repository. The flag ``-e`` is for editable.
 
@@ -19,7 +19,7 @@ This command compiles the C code and add statsmodels to your activate python
 environment by creating links from your python environment's libraries
 to the statsmodels source code. Therefore, changes to pure python code will
 be immediately available to the user without a re-install. Changes to C code or
-Cython code require rerunning ``pip install -e .`` before these changes are
+Cython code require rerunning ``python -m pip install -e ".[develop]"`` before these changes are
 available.
 
 Test Driven Development
@@ -37,7 +37,7 @@ Like many packages, statsmodels uses the `pytest testing system <https://docs.py
 Running Tests
 -------------
 Test are run from the command line by calling ``pytest``. Directly running tests using
-pytest requires that statsmodels is installed using ``pip install -e .`` as described
+pytest requires that statsmodels is installed using ``python -m pip install -e ".[develop]"`` as described
 above.
 
 Tests can be run at different levels of granularity:
diff --git a/statsmodels/sandbox/distributions/tests/test_multivariate.py b/statsmodels/sandbox/distributions/tests/test_multivariate.py
--- a/statsmodels/sandbox/distributions/tests/test_multivariate.py
+++ b/statsmodels/sandbox/distributions/tests/test_multivariate.py
@@ -4,8 +4,7 @@
 @author: Josef Perktold
 """
 import numpy as np
-from numpy.testing import (assert_almost_equal, assert_array_almost_equal,
-                           assert_allclose)
+from numpy.testing import assert_almost_equal,  assert_allclose
 
 from statsmodels.sandbox.distributions.multivariate import (
     mvstdtprob, mvstdnormcdf)
@@ -128,16 +127,21 @@ def test_mvn_pdf(self):
         cov3 = self.cov3
         mvn3 = self.mvn3
 
-        r_val = [-7.667977543898155, -6.917977543898155, -5.167977543898155]
-        assert_array_almost_equal(mvn3.logpdf(cov3), r_val, decimal=14)
-        # decimal 18
-        r_val = [0.000467562492721686, 0.000989829804859273,
-                 0.005696077243833402]
-        assert_array_almost_equal(mvn3.pdf(cov3), r_val, decimal=17)
+        r_val = [
+            -7.667977543898155, -6.917977543898155, -5.167977543898155
+        ]
+        assert_allclose(mvn3.logpdf(cov3), r_val, rtol=1e-13)
+
+        r_val = [
+            0.000467562492721686, 0.000989829804859273, 0.005696077243833402
+        ]
+        assert_allclose(mvn3.pdf(cov3), r_val, rtol=1e-13)
 
         mvn3b = MVNormal(np.array([0, 0, 0]), cov3)
-        r_val = [0.02914269740502042, 0.02269635555984291, 0.01767593948287269]
-        assert_array_almost_equal(mvn3b.pdf(cov3), r_val, decimal=16)
+        r_val = [
+            0.02914269740502042, 0.02269635555984291, 0.01767593948287269
+        ]
+        assert_allclose(mvn3b.pdf(cov3), r_val, rtol=1e-13)
 
     def test_mvt_pdf(self, reset_randomstate):
         cov3 = self.cov3
diff --git a/statsmodels/stats/tests/test_effectsize.py b/statsmodels/stats/tests/test_effectsize.py
--- a/statsmodels/stats/tests/test_effectsize.py
+++ b/statsmodels/stats/tests/test_effectsize.py
@@ -71,5 +71,6 @@ def test_noncent_t():
     mean = stats.nct.mean(df, res.nc)
     assert_allclose(t_stat, mean, rtol=1e-5)
 
+    # Tolerancee relaxed due to Boost integration in SciPy
     assert_allclose(stats.nct.cdf(t_stat, df, res.confint), [0.975, 0.025],
                     rtol=1e-5)
diff --git a/statsmodels/stats/tests/test_sandwich.py b/statsmodels/stats/tests/test_sandwich.py
--- a/statsmodels/stats/tests/test_sandwich.py
+++ b/statsmodels/stats/tests/test_sandwich.py
@@ -10,7 +10,7 @@
 Author: Josef Perktold
 """
 import numpy as np
-from numpy.testing import assert_almost_equal
+from numpy.testing import assert_almost_equal, assert_allclose
 
 from statsmodels.regression.linear_model import OLS
 from statsmodels.tools.tools import add_constant
@@ -18,73 +18,82 @@
 
 
 def test_cov_cluster_2groups():
-    #comparing cluster robust standard errors to Peterson
-    #requires Petersen's test_data
-    #http://www.kellogg.northwestern.edu/faculty/petersen/htm/papers/se/test_data.txt
+    # comparing cluster robust standard errors to Peterson
+    # requires Petersen's test_data
+    # http://www.kellogg.northwestern.edu/faculty/petersen
+    #      .../htm/papers/se/test_data.txt
     import os
     cur_dir = os.path.abspath(os.path.dirname(__file__))
-    fpath = os.path.join(cur_dir,"test_data.txt")
+    fpath = os.path.join(cur_dir, "test_data.txt")
     pet = np.genfromtxt(fpath)
-    endog = pet[:,-1]
-    group = pet[:,0].astype(int)
-    time = pet[:,1].astype(int)
-    exog = add_constant(pet[:,2])
+    endog = pet[:, -1]
+    group = pet[:, 0].astype(int)
+    time = pet[:, 1].astype(int)
+    exog = add_constant(pet[:, 2])
     res = OLS(endog, exog).fit()
 
     cov01, covg, covt = sw.cov_cluster_2groups(res, group, group2=time)
 
-    #Reference number from Petersen
-    #http://www.kellogg.northwestern.edu/faculty/petersen/htm/papers/se/test_data.htm
+    # Reference number from Petersen
+    # http://www.kellogg.northwestern.edu/faculty/petersen/htm
+    #       .../papers/se/test_data.htm
 
     bse_petw = [0.0284, 0.0284]
     bse_pet0 = [0.0670, 0.0506]
-    bse_pet1 = [0.0234, 0.0334]  #year
-    bse_pet01 = [0.0651, 0.0536]  #firm and year
+    bse_pet1 = [0.0234, 0.0334]  # year
+    bse_pet01 = [0.0651, 0.0536]  # firm and year
     bse_0 = sw.se_cov(covg)
     bse_1 = sw.se_cov(covt)
     bse_01 = sw.se_cov(cov01)
-    #print res.HC0_se, bse_petw - res.HC0_se
-    #print bse_0, bse_0 - bse_pet0
-    #print bse_1, bse_1 - bse_pet1
-    #print bse_01, bse_01 - bse_pet01
+    # print res.HC0_se, bse_petw - res.HC0_se
+    # print bse_0, bse_0 - bse_pet0
+    # print bse_1, bse_1 - bse_pet1
+    # print bse_01, bse_01 - bse_pet01
     assert_almost_equal(bse_petw, res.HC0_se, decimal=4)
     assert_almost_equal(bse_0, bse_pet0, decimal=4)
     assert_almost_equal(bse_1, bse_pet1, decimal=4)
     assert_almost_equal(bse_01, bse_pet01, decimal=4)
 
-def test_hac_simple():
 
+def test_hac_simple():
     from statsmodels.datasets import macrodata
     d2 = macrodata.load_pandas().data
-    g_gdp = 400*np.diff(np.log(d2['realgdp'].values))
-    g_inv = 400*np.diff(np.log(d2['realinv'].values))
+    g_gdp = 400 * np.diff(np.log(d2['realgdp'].values))
+    g_inv = 400 * np.diff(np.log(d2['realinv'].values))
     exogg = add_constant(np.c_[g_gdp, d2['realint'][:-1].values])
     res_olsg = OLS(g_inv, exogg).fit()
 
-
-
-    #> NeweyWest(fm, lag = 4, prewhite = FALSE, sandwich = TRUE, verbose=TRUE, adjust=TRUE)
-    #Lag truncation parameter chosen: 4
-    #                     (Intercept)                   ggdp                  lint
-    cov1_r = [[  1.40643899878678802, -0.3180328707083329709, -0.060621111216488610],
-             [ -0.31803287070833292,  0.1097308348999818661,  0.000395311760301478],
-             [ -0.06062111121648865,  0.0003953117603014895,  0.087511528912470993]]
-
-    #> NeweyWest(fm, lag = 4, prewhite = FALSE, sandwich = TRUE, verbose=TRUE, adjust=FALSE)
-    #Lag truncation parameter chosen: 4
-    #                    (Intercept)                  ggdp                  lint
-    cov2_r = [[ 1.3855512908840137, -0.313309610252268500, -0.059720797683570477],
-             [ -0.3133096102522685,  0.108101169035130618,  0.000389440793564339],
-             [ -0.0597207976835705,  0.000389440793564336,  0.086211852740503622]]
+    # > NeweyWest(fm, lag = 4, prewhite = FALSE, sandwich = TRUE,
+    #             verbose=TRUE, adjust=TRUE)
+    # Lag truncation parameter chosen: 4
+    #             (Intercept)                   ggdp                  lint
+    cov1_r = [
+        [+1.40643899878678802, -0.3180328707083329709, -0.060621111216488610],
+        [-0.31803287070833292, 0.1097308348999818661, +0.000395311760301478],
+        [-0.06062111121648865, 0.0003953117603014895, +0.087511528912470993]
+    ]
+
+    # > NeweyWest(fm, lag = 4, prewhite = FALSE, sandwich = TRUE,
+    #             verbose=TRUE, adjust=FALSE)
+    # Lag truncation parameter chosen: 4
+    #         (Intercept)                  ggdp                  lint
+    cov2_r = [
+        [+1.3855512908840137, -0.313309610252268500, -0.059720797683570477],
+        [-0.3133096102522685, +0.108101169035130618, +0.000389440793564339],
+        [-0.0597207976835705, +0.000389440793564336, +0.086211852740503622]
+    ]
 
     cov1 = sw.cov_hac_simple(res_olsg, nlags=4, use_correction=True)
-    se1 =  sw.se_cov(cov1)
+    se1 = sw.se_cov(cov1)
     cov2 = sw.cov_hac_simple(res_olsg, nlags=4, use_correction=False)
-    se2 =  sw.se_cov(cov2)
-    assert_almost_equal(cov1, cov1_r, decimal=8)
-    assert_almost_equal(cov2, cov2_r, decimal=8)
+    se2 = sw.se_cov(cov2)
+    # Relax precision requirements for this test due to failure in NumPy 1.23
+    assert_allclose(cov1, cov1_r)
+    assert_allclose(cov2, cov2_r)
+    assert_allclose(np.sqrt(np.diag(cov1_r)), se1)
+    assert_allclose(np.sqrt(np.diag(cov2_r)), se2)
 
     # compare default for nlags
     cov3 = sw.cov_hac_simple(res_olsg, use_correction=False)
     cov4 = sw.cov_hac_simple(res_olsg, nlags=4, use_correction=False)
-    assert_almost_equal(cov3, cov4, decimal=8)
+    assert_allclose(cov3, cov4)