Repository: bashtage/linearmodels
Pull request #577 (2024-01-04 10:57:00)

CI: Formalize pip pre testing


Patch:
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index e8e2bfee8..000000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-skip_tags: true
-clone_depth: 50
-
-image:
-  - Visual Studio 2019
-
-matrix:
-    fast_finish: false
-
-environment:
-  global:
-    OMP_NUM_THREADS: 1
-    MKL_NUM_THREADS: 1
-    VML_NUM_THREADS: 1
-    PYTHONHASHSEED: 31211999
-
-  matrix:
-    - PYTHON: C:\Python311-x64
-      PYTHON_VERSION: 3.11
-      PYTHON_ARCH: 64
-
-
-# We always use a 64-bit machine, but can build x86 distributions
-# with the TARGET_ARCH variable.
-platform:
-    - x64
-
-install:
-    - git submodule update --init --recursive
-    - git fetch --all --tags
-
-    # Set path
-    - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
-
-    # Check that we have the expected version and architecture for Python
-    - python --version
-    - python -m pip install --upgrade pip wheel
-
-build_script:
-    # Install build requirements
-    - pip install -r requirements.txt
-    - pip install -r requirements-test.txt
-    - python -m pip wheel --wheel-dir dist .
-    - ls dist/*
-
-test_script:
-    # install from wheel
-    - pip install --no-index --find-links dist/ linearmodels
-
-    # run tests from install wheel
-    - mkdir _for_testing
-    - cd _for_testing
-    - python -c "import linearmodels; linearmodels.test(['-n 2','--skip-slow','--skip-smoke','--durations=25'])"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index d2105166a..81370b5e9 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -10,10 +10,11 @@ variables:
   VML_NUM_THREADS: 1
   OPENBLAS_NUM_THREADS: 1
   PYTHONHASHSEED: 12345678 # Ensure tests are correctly gathered by xdist
-  SETUPTOOLS_USE_DISTUTILS: "stdlib"
   TEST_INSTALL: false
   MPLBACKEND: agg
   coverage: true
+  test.install: false
+  pip.pre: false
 
 jobs:
 - template: ci/azure_template_posix.yml
diff --git a/ci/azure_template_posix.yml b/ci/azure_template_posix.yml
index 234448f30..060439c94 100644
--- a/ci/azure_template_posix.yml
+++ b/ci/azure_template_posix.yml
@@ -56,14 +56,21 @@ jobs:
         XARRAY: 22.0
       python310_latest:
         python.version: '3.10'
+        FORMULAIC: 1.0.1
         XXHASH: true
       python311_latest:
         python.version: '3.11'
         XXHASH: true
-      python311_copy_on_write:
-        python.version: '3.11'
+      python312_latest:
+        python.version: '3.12'
+        XXHASH: true
+      python312_copy_on_write:
+        python.version: '3.12'
         XXHASH: true
         LM_TEST_COPY_ON_WRITE: 1
+      python312_pre:
+        python.version: '3.12'
+        pip.pre: true
     maxParallel: 10
 
   steps:
@@ -113,7 +120,7 @@ jobs:
       python -c "import linearmodels; linearmodels.test(['-n', 'auto', '--junitxml=../junit/test-results.xml'])"
       popd
     displayName: 'Run tests (site-packages)'
-    condition: eq(variables['test.install'], 'true')
+    condition: and(eq(variables['test.install'], 'true'), ne(variables['pip.pre'], 'true'))
 
   - script: |
       echo "Testing editable install"
@@ -123,7 +130,15 @@ jobs:
       echo pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} linearmodels/tests
       pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} linearmodels/tests
     displayName: 'Run tests (editable)'
-    condition: ne(variables['test.install'], 'true')
+    condition: and(ne(variables['test.install'], 'true'), ne(variables['pip.pre'], 'true'))
+
+  - script: |
+      echo "Testing pip-pre"
+      echo pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} linearmodels/tests
+      pytest -m "${PYTEST_PATTERN}" --junitxml=junit/test-results.xml -n auto --durations=25 ${COVERAGE_OPTS} linearmodels/tests
+    displayName: 'Run tests (pip pre)'
+    condition: eq(variables['pip.pre'], 'true')
+    continueOnError: true
 
   - task: PublishTestResults@2
     inputs:
diff --git a/ci/install-posix.sh b/ci/install-posix.sh
index 06073d6b3..11dc3272a 100644
--- a/ci/install-posix.sh
+++ b/ci/install-posix.sh
@@ -13,3 +13,9 @@ if [[ -n ${FORMULAIC} ]]; then CMD="$CMD formulaic~=${FORMULAIC}"; fi
 if [[ -n ${XXHASH} ]]; then CMD="$CMD xxhash"; fi
 echo "$CMD"
 eval "$CMD"
+
+if [ "${PIP_PRE}" = true ]; then
+  python -m pip uninstall -y numpy pandas scipy matplotlib statsmodels
+  python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy pandas scipy matplotlib --upgrade --use-deprecated=legacy-resolver
+  python -m pip install git+https://github.com/statsmodels/statsmodels.git --upgrade --no-build-isolation -v
+fi
\ No newline at end of file
diff --git a/doc/make.bat b/doc/make.bat
index 0b49d06db..8274ced54 100644
--- a/doc/make.bat
+++ b/doc/make.bat
@@ -11,6 +11,7 @@ set SOURCEDIR=source
 set BUILDDIR=build
 set SPHINXPROJ=linearmodels
 set SPHINXOPTS=
+set PYDEVD_DISABLE_FILE_VALIDATION=1
 
 if "%1" == "" goto help
 
@@ -23,15 +24,18 @@ if errorlevel 9009 (
 	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
+	echo.https://sphinx-doc.org/
+	set PYDEVD_DISABLE_FILE_VALIDATION=
 	exit /b 1
 )
 
 %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+set PYDEVD_DISABLE_FILE_VALIDATION=
 goto end
 
 :help
 %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+set PYDEVD_DISABLE_FILE_VALIDATION=
 
 :end
 popd
diff --git a/doc/requirements.txt b/doc/requirements.txt
index c88ae924b..6b3db178f 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -11,4 +11,4 @@ nbformat
 matplotlib
 seaborn
 mypy_extensions>=0.4
-mistune<2
\ No newline at end of file
+# mistune<2
\ No newline at end of file
diff --git a/doc/source/changes/5.0.rst b/doc/source/changes/5.0.rst
index 89b19c0b0..e7833d557 100644
--- a/doc/source/changes/5.0.rst
+++ b/doc/source/changes/5.0.rst
@@ -1,3 +1,8 @@
+Verion 5.4
+----------
+- Compatability with NumPy 2
+- Compatability with recent pandas releases
+
 Version 5.3
 -----------
 - Bumped the minimum formulaic to 0.6.5.
diff --git a/linearmodels/compat/pandas.py b/linearmodels/compat/pandas.py
new file mode 100644
index 000000000..3b424f218
--- /dev/null
+++ b/linearmodels/compat/pandas.py
@@ -0,0 +1,9 @@
+from packaging.version import parse
+import pandas as pd
+
+PANDAS_VERSION = parse(pd.__version__)
+PD_GTE_21 = not (PANDAS_VERSION <= parse("2.0.99"))
+PD_LT_22 = PANDAS_VERSION <= parse("2.1.99")
+ANNUAL_FREQ = "A-DEC" if PD_LT_22 else "YE-DEC"
+
+__all__ = ["ANNUAL_FREQ", "PD_GTE_21", "PD_LT_22"]
diff --git a/linearmodels/iv/results.py b/linearmodels/iv/results.py
index 663ecdc9c..bde9620d2 100644
--- a/linearmodels/iv/results.py
+++ b/linearmodels/iv/results.py
@@ -764,7 +764,11 @@ class FirstStageResults(_SummaryStr):
         ]
         out_df = out_df[cols]
         for col in out_df:
-            out_df[col] = to_numeric(out_df[col], errors="ignore")
+            try:
+                out_df[col] = to_numeric(out_df[col])
+            except ValueError:
+                # If an error is raised, ignore and keep the column
+                pass
 
         return out_df
 
diff --git a/linearmodels/panel/data.py b/linearmodels/panel/data.py
index 64e7d4cf4..153bdff27 100644
--- a/linearmodels/panel/data.py
+++ b/linearmodels/panel/data.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from linearmodels.compat.pandas import PD_GTE_21
+
 from collections.abc import Hashable, Sequence
 from itertools import product
 from typing import Literal, Union, cast, overload
@@ -227,7 +229,8 @@ class PanelData:
                 if copy:
                     self._frame = self._frame.copy()
             else:
-                self._frame = DataFrame({var_name: x.T.stack(dropna=False)})
+                options = {"future_stack": True} if PD_GTE_21 else {"dropna": False}
+                self._frame = DataFrame({var_name: x.T.stack(**options)})
         elif isinstance(x, np.ndarray):
             if x.ndim not in (2, 3):
                 raise ValueError("2 or 3-d array required for numpy input")
@@ -298,6 +301,8 @@ class PanelData:
             Boolean array indicating observations to drop with reference to
             the dataframe view of the data
         """
+        if isinstance(locs, Series):
+            locs = np.asarray(locs)
         self._frame = self._frame.loc[~locs.ravel()]
         self._frame = self._minimize_multiindex(self._frame)
         # Reset panel and shape after a drop
diff --git a/linearmodels/panel/model.py b/linearmodels/panel/model.py
index 66e22b21d..15539992f 100644
--- a/linearmodels/panel/model.py
+++ b/linearmodels/panel/model.py
@@ -1878,7 +1878,9 @@ class PanelOLS(_PanelModelBase):
                     if self._constant:
                         assert isinstance(self._constant_index, int)
                         self._constant_index = int(
-                            np.argwhere(np.array(retain) == self._constant_index)
+                            np.squeeze(
+                                np.argwhere(np.array(retain) == self._constant_index)
+                            )
                         )
 
                     # Adjust exog
diff --git a/linearmodels/panel/utility.py b/linearmodels/panel/utility.py
index 1aa7a5a78..ec5a1c805 100644
--- a/linearmodels/panel/utility.py
+++ b/linearmodels/panel/utility.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from linearmodels.compat.pandas import ANNUAL_FREQ
+
 from collections import defaultdict
 from typing import NamedTuple, TypeVar, cast
 
@@ -613,7 +615,7 @@ def generate_panel_data(
         x.flat[locs] = np.nan
 
     entities = [f"firm{i}" for i in range(n)]
-    time = [dt for dt in date_range("1-1-1900", periods=t, freq="A-DEC")]
+    time = [dt for dt in date_range("1-1-1900", periods=t, freq=ANNUAL_FREQ)]
     var_names = [f"x{i}" for i in range(k)]
     if const:
         var_names[1:] = var_names[:-1]
diff --git a/linearmodels/shared/hypotheses.py b/linearmodels/shared/hypotheses.py
index 18a55cc21..05b61c59b 100644
--- a/linearmodels/shared/hypotheses.py
+++ b/linearmodels/shared/hypotheses.py
@@ -117,13 +117,13 @@ class InvalidTestStatistic(WaldTestStatistic):
 
     def __init__(self, reason: str, *, name: str | None = None) -> None:
         self._reason = reason
-        super().__init__(np.NaN, "", df=1, df_denom=1, name=name)
+        super().__init__(np.nan, "", df=1, df_denom=1, name=name)
         self.dist_name = "None"
 
     @property
     def pval(self) -> float:
-        """Always returns np.NaN"""
-        return np.NaN
+        """Always returns np.nan"""
+        return np.nan
 
     @property
     def critical_values(self) -> None:
@@ -158,13 +158,13 @@ class InapplicableTestStatistic(WaldTestStatistic):
         if reason is None:
             self._reason = "Test is not applicable to model specification"
 
-        super().__init__(np.NaN, "", df=1, df_denom=1, name=name)
+        super().__init__(np.nan, "", df=1, df_denom=1, name=name)
         self.dist_name = "None"
 
     @property
     def pval(self) -> float:
-        """Always returns np.NaN"""
-        return np.NaN
+        """Always returns np.nan"""
+        return np.nan
 
     @property
     def critical_values(self) -> None:
diff --git a/pyproject.toml b/pyproject.toml
index 5d0d5e529..3426fa102 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ requires = [
   "wheel",
   "setuptools_scm[toml]>=7,<8",
   "oldest-supported-numpy",
-  "numpy; python_version>='3.12'",
+  "numpy; python_version>='3.13'",
   "cython>=0.29.34"
 ]
 build-backend = "setuptools.build_meta"


Test patch:
diff --git a/linearmodels/tests/panel/_utility.py b/linearmodels/tests/panel/_utility.py
index bbd771d38..9825f4c96 100644
--- a/linearmodels/tests/panel/_utility.py
+++ b/linearmodels/tests/panel/_utility.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from linearmodels.compat.pandas import ANNUAL_FREQ
+
 from typing import Literal
 
 import numpy as np
@@ -121,7 +123,7 @@ def generate_data(
         return AttrDict(y=y, x=x, w=w, c=c, vc1=vc1, vc2=vc2)
 
     entities = ["firm" + str(i) for i in range(n)]
-    time = date_range("1-1-1900", periods=t, freq="A-DEC")
+    time = date_range("1-1-1900", periods=t, freq=ANNUAL_FREQ)
     var_names = ["x" + str(i) for i in range(k)]
     # y = DataFrame(y, index=time, columns=entities)
     y_df = panel_to_frame(
diff --git a/linearmodels/tests/shared/test_utility.py b/linearmodels/tests/shared/test_utility.py
index 76f383d39..abe6ae762 100644
--- a/linearmodels/tests/shared/test_utility.py
+++ b/linearmodels/tests/shared/test_utility.py
@@ -1,3 +1,5 @@
+from linearmodels.compat.pandas import ANNUAL_FREQ
+
 import pickle
 import random
 import string
@@ -224,7 +226,7 @@ def test_panel_to_midf():
             ],
         )
     )
-    times = pd.date_range("1999-12-31", freq="A-DEC", periods=7)
+    times = pd.date_range("1999-12-31", freq=ANNUAL_FREQ, periods=7)
     var_names = [f"x.{i}" for i in range(1, 4)]
     df3 = panel_to_frame(x, var_names, times, entities, True)
     mi = pd.MultiIndex.from_product([times, entities])
diff --git a/requirements-test.txt b/requirements-test.txt
index 9011e87a5..f05e387f5 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -2,6 +2,7 @@ black[jupyter]==23.11.0
 coverage
 flake8
 isort
+colorama
 matplotlib
 pytest>=7.3.0
 pytest-xdist