Repository: Pyomo/pyomo

Issue:
PyNumero relies on deprecated APIs
## Summary

PyNumero relies on 2 functions from `scipy.sparse.sputils` that have been deprecated (moved to a private module).

In addition, PyNumero relies on the NumPy `matrix` subclass, which "is not the recommended way to represent matrices or deal with linear algebra (see https://docs.scipy.org/doc/numpy/user/numpy-for-matlab-users.html)."  This interface should be updated to use regular `ndarray`.

### Steps to reproduce the issue

See `_sputils` imports in 
- pyomo/contrib/pynumero/sparse/block_matrix.py
- pyomo/contrib/pynumero/sparse/utils.py

See the warning emitted by `pytest pyomo/contrib/pynumero`:
```
pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py: 50 warnings
  pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py:694: PendingDeprecationWarning:

  the matrix subclass is not the recommended way to represent matrices or deal with linear algebra (see https://docs.scipy.org/doc/numpy/user/numpy-for-matlab-users.html). Please adjust your code to use regular ndarray.
```

### Error Message

(see above)

### Information on your system

Pyomo version: 6.6.2.dev0
Python version: all (testing on 3.11)
Operating system: all (testing on Linux)
How Pyomo was installed (PyPI, conda, source): source
Solver (if applicable): n/a


### Additional information

PR #2897 silenced the `sputils` deprecation messages (so that the import tests would pass)




Discussion:

Resolving patch:
diff --git a/pyomo/contrib/pynumero/sparse/block_matrix.py b/pyomo/contrib/pynumero/sparse/block_matrix.py
--- a/pyomo/contrib/pynumero/sparse/block_matrix.py
+++ b/pyomo/contrib/pynumero/sparse/block_matrix.py
@@ -22,7 +22,6 @@
 
 """
 
-from scipy.sparse.sputils import get_index_dtype
 from pyomo.contrib.pynumero.sparse.block_vector import BlockVector
 from scipy.sparse import coo_matrix, csr_matrix, csc_matrix
 from scipy.sparse import isspmatrix
@@ -393,9 +392,8 @@ def tocoo(self, copy=True):
 
         # create pointers for COO matrix (row, col, data)
         data = np.empty(nonzeros, dtype=dtype)
-        idx_dtype = get_index_dtype(maxval=max(shape))
-        row = -np.ones(nonzeros, dtype=idx_dtype)
-        col = -np.ones(nonzeros, dtype=idx_dtype)
+        row = -np.ones(nonzeros, dtype=int)
+        col = -np.ones(nonzeros, dtype=int)
 
         # populate COO pointers
         nnz = 0
diff --git a/pyomo/contrib/pynumero/sparse/utils.py b/pyomo/contrib/pynumero/sparse/utils.py
deleted file mode 100644
--- a/pyomo/contrib/pynumero/sparse/utils.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#  ___________________________________________________________________________
-#
-#  Pyomo: Python Optimization Modeling Objects
-#  Copyright (c) 2008-2022
-#  National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
-#  rights in this software.
-#  This software is distributed under the 3-clause BSD License.
-#  ___________________________________________________________________________
-from scipy.sparse.sputils import isscalarlike
-from scipy.sparse import tril, triu, isspmatrix
-
-import numpy as np
-
-
-def is_symmetric_dense(mat):
-    flag = False
-    if isinstance(mat, np.ndarray):
-        if mat.ndim == 2 and mat.shape[0] == mat.shape[1]:
-            if np.allclose(mat, mat.T, atol=1e-6):
-                flag = True
-    elif isscalarlike(mat):
-        flag = True
-    else:
-        raise RuntimeError("Format not recognized {}".format(type(mat)))
-    return flag
-
-
-def is_symmetric_sparse(mat):
-    from pyomo.contrib.pynumero.sparse.block_matrix import BlockMatrix
-
-    # Note: this check is expensive
-    flag = False
-    if isinstance(mat, np.ndarray):
-        flag = is_symmetric_dense(mat)
-    elif isscalarlike(mat):
-        flag = True
-    elif isspmatrix(mat) or isinstance(mat, BlockMatrix):
-        if mat.shape[0] != mat.shape[1]:
-            flag = False
-        else:
-            if isinstance(mat, BlockMatrix):
-                mat = mat.tocoo()
-            # get upper and lower triangular
-            l = tril(mat)
-            u = triu(mat)
-            diff = l - u.transpose()
-            z = np.zeros(diff.nnz)
-            flag = np.allclose(diff.data, z, atol=1e-6)
-    else:
-        raise RuntimeError("Format not recognized {}".format(type(mat)))
-    return flag


Test patch:
diff --git a/pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py b/pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py
--- a/pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py
+++ b/pyomo/contrib/pynumero/interfaces/tests/test_external_pyomo_model.py
@@ -691,7 +691,6 @@ def test_external_hessian_Model2by2(self):
             hess = external_model.evaluate_hessian_external_variables()
             expected_hess = model.evaluate_external_hessian(x)
             for matrix1, matrix2 in zip(hess, expected_hess):
-                matrix2 = np.matrix(matrix2)
                 np.testing.assert_allclose(matrix1, matrix2, rtol=1e-8)
 
     def test_external_jacobian_SimpleModel2x2_1(self):
diff --git a/pyomo/contrib/pynumero/sparse/tests/test_sparse_utils.py b/pyomo/contrib/pynumero/sparse/tests/test_sparse_utils.py
deleted file mode 100644
--- a/pyomo/contrib/pynumero/sparse/tests/test_sparse_utils.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#  ___________________________________________________________________________
-#
-#  Pyomo: Python Optimization Modeling Objects
-#  Copyright (c) 2008-2022
-#  National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
-#  rights in this software.
-#  This software is distributed under the 3-clause BSD License.
-#  ___________________________________________________________________________
-import pyomo.common.unittest as unittest
-
-from pyomo.contrib.pynumero.dependencies import (
-    numpy as np,
-    numpy_available,
-    scipy_available,
-)
-
-if not (numpy_available and scipy_available):
-    raise unittest.SkipTest("Pynumero needs scipy and numpy to run NLP tests")
-
-from scipy.sparse import coo_matrix, bmat
-
-from pyomo.contrib.pynumero.sparse.utils import is_symmetric_dense, is_symmetric_sparse
-
-
-class TestSparseUtils(unittest.TestCase):
-    def setUp(self):
-        row = np.array([0, 1, 4, 1, 2, 7, 2, 3, 5, 3, 4, 5, 4, 7, 5, 6, 6, 7])
-        col = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 7])
-        data = np.array(
-            [27, 5, 12, 56, 66, 34, 94, 31, 41, 7, 98, 72, 24, 33, 78, 47, 98, 41]
-        )
-
-        off_diagonal_mask = row != col
-        new_row = np.concatenate([row, col[off_diagonal_mask]])
-        new_col = np.concatenate([col, row[off_diagonal_mask]])
-        new_data = np.concatenate([data, data[off_diagonal_mask]])
-        m = coo_matrix((new_data, (new_row, new_col)), shape=(8, 8))
-
-        self.block00 = m
-
-        row = np.array([0, 3, 1, 0])
-        col = np.array([0, 3, 1, 2])
-        data = np.array([4, 5, 7, 9])
-        m = coo_matrix((data, (row, col)), shape=(4, 8))
-
-        self.block10 = m
-
-        row = np.array([0, 1, 2, 3])
-        col = np.array([0, 1, 2, 3])
-        data = np.array([1, 1, 1, 1])
-        m = coo_matrix((data, (row, col)), shape=(4, 4))
-
-        self.block11 = m
-
-    def test_is_symmetric_dense(self):
-        m = self.block00.toarray()
-        self.assertTrue(is_symmetric_dense(m))
-        self.assertTrue(is_symmetric_dense(2))
-        with self.assertRaises(Exception) as context:
-            self.assertTrue(is_symmetric_dense(self.block00))
-
-    def test_is_symmetric_sparse(self):
-        m = self.block00
-        self.assertTrue(is_symmetric_sparse(m))
-        m = self.block00.toarray()
-        self.assertTrue(is_symmetric_sparse(m))
-        m = self.block11
-        self.assertTrue(is_symmetric_sparse(m))
-        m = self.block10
-        self.assertFalse(is_symmetric_sparse(m))
-        self.assertTrue(is_symmetric_sparse(2))
-
-        row = np.array([0, 1, 2, 3])
-        col = np.array([0, 1, 2, 3])
-        data = np.array([1, 1, 1, 1])
-        m = coo_matrix((data, (row, col)), shape=(4, 6))
-        self.assertFalse(is_symmetric_sparse(m))
-
-        with self.assertRaises(Exception) as context:
-            self.assertTrue(is_symmetric_sparse(range(5)))