Repository: Pyomo/pyomo

Issue:
cyipopt v1.0.3 Release Issue
A new version of `cyipopt` was released today, and it is now causing both a warning message and a failure:

```
C:\Miniconda\envs\test\lib\site-packages\ipopt\__init__.py:13: FutureWarning: The module has been renamed to 'cyipopt' from 'ipopt'. 
Please import using 'import cyipopt' and remove all uses of 'import ipopt' in your code as this will be deprecated
 in a future release.

======================================================================
ERROR: test_status_maps (pyomo.contrib.pynumero.tests.test_cyipopt_examples.TestPyomoCyIpoptSolver)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/runner/work/pyomo/pyomo/pyomo/contrib/pynumero/tests/test_cyipopt_examples.py", line 48, in test_status_maps
    self.assertEqual(len(cyipopt_core.STATUS_MESSAGES),
AttributeError: module 'cyipopt' has no attribute 'STATUS_MESSAGES'
```



Discussion:

Resolving patch:
diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py
--- a/pyomo/common/dependencies.py
+++ b/pyomo/common/dependencies.py
@@ -409,12 +409,12 @@ def attempt_import(name, error_message=None, only_catch_importerror=None,
             error_message = "The %s module version %s does not satisfy " \
                             "the minimum version %s" % (
                                 name, version, minimum_version)
-    except catch_exceptions:
-        pass
+    except catch_exceptions as e:
+        _err = str(e)
 
     if not error_message:
         error_message = "The %s module (an optional Pyomo dependency) " \
-                        "failed to import" % (name,)
+                        "failed to import: %s" % (name, _err)
 
     module = ModuleUnavailable(error_message)
     if callback is not None:
diff --git a/pyomo/contrib/pynumero/algorithms/solvers/cyipopt_solver.py b/pyomo/contrib/pynumero/algorithms/solvers/cyipopt_solver.py
--- a/pyomo/contrib/pynumero/algorithms/solvers/cyipopt_solver.py
+++ b/pyomo/contrib/pynumero/algorithms/solvers/cyipopt_solver.py
@@ -10,7 +10,7 @@
 """
 The cyipopt_solver module includes the python interface to the
 Cythonized ipopt solver cyipopt (see more:
-https://github.com/matthias-k/cyipopt.git). To use the solver,
+https://github.com/mechmotum/cyipopt.git). To use the solver,
 you can create a derived implementation from the abstract base class
 CyIpoptProblemInterface that provides the necessary methods.
 
@@ -28,12 +28,32 @@
     attempt_import,
     numpy as np, numpy_available,
 )
-ipopt, ipopt_available = attempt_import(
-    'ipopt',
-    error_message='cyipopt solver relies on the ipopt module from cyipopt. '
-    'See https://github.com/matthias-k/cyipopt.git for cyipopt '
-    'installation instructions.'
+
+def _cyipopt_importer():
+    import cyipopt
+    # cyipopt before version 1.0.3 called the problem class "Problem"
+    if not hasattr(cyipopt, 'Problem'):
+        cyipopt.Problem = cyipopt.problem
+    # cyipopt before version 1.0.3 put the __version__ flag in the ipopt
+    # module (which was deprecated starting in 1.0.3)
+    if not hasattr(cyipopt, '__version__'):
+        import ipopt
+        cyipopt.__version__ = ipopt.__version__
+    # Beginning in 1.0.3, STATUS_MESSAGES is in a separate
+    # ipopt_wrapper module
+    if not hasattr(cyipopt, 'STATUS_MESSAGES'):
+        import ipopt_wrapper
+        cyipopt.STATUS_MESSAGES = ipopt_wrapper.STATUS_MESSAGES
+    return cyipopt
+
+cyipopt, cyipopt_available = attempt_import(
+     'ipopt',
+     error_message='cyipopt solver relies on the ipopt module from cyipopt. '
+     'See https://github.com/mechmotum/cyipopt.git for cyipopt '
+     'installation instructions.',
+     importer=_cyipopt_importer,
 )
+
 # Because pynumero.interfaces requires numpy, we will leverage deferred
 # imports here so that the solver can be registered even when numpy is
 # not available.
@@ -53,25 +73,48 @@
 # This maps the cyipopt STATUS_MESSAGES back to string representations
 # of the Ipopt ApplicationReturnStatus enum
 _cyipopt_status_enum = [
-    'Solve_Succeeded', b'Algorithm terminated successfully at a locally optimal point, satisfying the convergence tolerances (can be specified by options).',
-    'Solved_To_Acceptable_Level', b'Algorithm stopped at a point that was converged, not to "desired" tolerances, but to "acceptable" tolerances (see the acceptable-... options).',
-    'Infeasible_Problem_Detected', b'Algorithm converged to a point of local infeasibility. Problem may be infeasible.',
-    'Search_Direction_Becomes_Too_Small', b'Algorithm proceeds with very little progress.',
-    'Diverging_Iterates', b'It seems that the iterates diverge.',
-    'User_Requested_Stop', b'The user call-back function intermediate_callback (see Section 3.3.4 in the documentation) returned false, i.e., the user code requested a premature termination of the optimization.',
-    'Feasible_Point_Found', b'Feasible point for square problem found.',
-    'Maximum_Iterations_Exceeded', b'Maximum number of iterations exceeded (can be specified by an option).',
-    'Restoration_Failed', b'Restoration phase failed, algorithm doesn\'t know how to proceed.',
-    'Error_In_Step_Computation', b'An unrecoverable error occurred while Ipopt tried to compute the search direction.',
-    'Maximum_CpuTime_Exceeded', b'Maximum CPU time exceeded.',
-    'Not_Enough_Degrees_Of_Freedom', b'Problem has too few degrees of freedom.',
-    'Invalid_Problem_Definition', b'Invalid problem definition.',
-    'Invalid_Option', b'Invalid option encountered.',
-    'Invalid_Number_Detected', b'Algorithm received an invalid number (such as NaN or Inf) from the NLP; see also option check_derivatives_for_naninf',
-    'Unrecoverable_Exception', b'Some uncaught Ipopt exception encountered.',
-    'NonIpopt_Exception_Thrown', b'Unknown Exception caught in Ipopt',
-    'Insufficient_Memory', b'Not enough memory.',
-    'Internal_Error', b'An unknown internal error occurred. Please contact the Ipopt authors through the mailing list.'
+    "Solve_Succeeded", (b"Algorithm terminated successfully at a locally "
+                        b"optimal point, satisfying the convergence tolerances "
+                        b"(can be specified by options)."),
+    "Solved_To_Acceptable_Level", (b"Algorithm stopped at a point that was "
+                                   b"converged, not to \"desired\" tolerances, "
+                                   b"but to \"acceptable\" tolerances (see the "
+                                   b"acceptable-... options)."),
+    "Infeasible_Problem_Detected", (b"Algorithm converged to a point of local "
+                                    b"infeasibility. Problem may be "
+                                    b"infeasible."),
+    "Search_Direction_Becomes_Too_Small", (b"Algorithm proceeds with very "
+                                           b"little progress."),
+    "Diverging_Iterates", b"It seems that the iterates diverge.",
+    "User_Requested_Stop", (b"The user call-back function intermediate_callback "
+                            b"(see Section 3.3.4 in the documentation) returned "
+                            b"false, i.e., the user code requested a premature "
+                            b"termination of the optimization."),
+    "Feasible_Point_Found", b"Feasible point for square problem found.",
+    "Maximum_Iterations_Exceeded", (b"Maximum number of iterations exceeded "
+                                    b"(can be specified by an option)."),
+    "Restoration_Failed", (b"Restoration phase failed, algorithm doesn\'t know "
+                           b"how to proceed."),
+    "Error_In_Step_Computation", (b"An unrecoverable error occurred while Ipopt "
+                                  b"tried to compute the search direction."),
+    "Maximum_CpuTime_Exceeded", b"Maximum CPU time exceeded.",
+    "Not_Enough_Degrees_Of_Freedom", b"Problem has too few degrees of freedom.",
+    "Invalid_Problem_Definition", b"Invalid problem definition.",
+    "Invalid_Option", b"Invalid option encountered.",
+    "Invalid_Number_Detected", (b"Algorithm received an invalid number (such as "
+                                b"NaN or Inf) from the NLP; see also option "
+                                b"check_derivatives_for_naninf."),
+    # Note that the concluding "." was missing before cyipopt 1.0.3
+    "Invalid_Number_Detected", (b"Algorithm received an invalid number (such as "
+                                b"NaN or Inf) from the NLP; see also option "
+                                b"check_derivatives_for_naninf"),
+    "Unrecoverable_Exception", b"Some uncaught Ipopt exception encountered.",
+    "NonIpopt_Exception_Thrown", b"Unknown Exception caught in Ipopt.",
+    # Note that the concluding "." was missing before cyipopt 1.0.3
+    "NonIpopt_Exception_Thrown", b"Unknown Exception caught in Ipopt",
+    "Insufficient_Memory", b"Not enough memory.",
+    "Internal_Error", (b"An unknown internal error occurred. Please contact "
+                       b"the Ipopt authors through the mailing list."),
 ]
 _cyipopt_status_enum = {
     _cyipopt_status_enum[i+1]: _cyipopt_status_enum[i]
@@ -396,7 +439,7 @@ def solve(self, x0=None, tee=False):
         nx = len(xstart)
         ng = len(gl)
 
-        cyipopt_solver = ipopt.problem(
+        cyipopt_solver = cyipopt.Problem(
             n=nx,
             m=ng,
             problem_obj=self._problem,
@@ -480,13 +523,13 @@ def _set_model(self, model):
         self._model = model
 
     def available(self, exception_flag=False):
-        return numpy_available and ipopt_available
+        return numpy_available and cyipopt_available
 
     def license_is_valid(self):
         return True
 
     def version(self):
-        return tuple(int(_) for _ in ipopt.__version__.split('.'))
+        return tuple(int(_) for _ in cyipopt.__version__.split('.'))
 
     def solve(self, model, **kwds):
         config = self.config(kwds, preserve_implicit=True)
@@ -515,7 +558,7 @@ def solve(self, model, **kwds):
         nx = len(xl)
         ng = len(gl)
 
-        cyipopt_solver = ipopt.problem(
+        cyipopt_solver = cyipopt.Problem(
             n=nx,
             m=ng,
             problem_obj=problem,


Test patch:
diff --git a/pyomo/common/tests/test_dependencies.py b/pyomo/common/tests/test_dependencies.py
--- a/pyomo/common/tests/test_dependencies.py
+++ b/pyomo/common/tests/test_dependencies.py
@@ -230,16 +230,18 @@ def test_generate_warning(self):
         log = StringIO()
         with LoggingIntercept(log, 'pyomo.common'):
             mod.generate_import_warning()
-        self.assertEqual(
-            log.getvalue(), "The pyomo.common.tests.dep_mod_except module "
-            "(an optional Pyomo dependency) failed to import\n")
+        self.assertIn(
+            "The pyomo.common.tests.dep_mod_except module "
+            "(an optional Pyomo dependency) failed to import",
+            log.getvalue())
 
         log = StringIO()
         with LoggingIntercept(log, 'pyomo.core.base'):
             mod.generate_import_warning('pyomo.core.base')
-        self.assertEqual(
-            log.getvalue(), "The pyomo.common.tests.dep_mod_except module "
-            "(an optional Pyomo dependency) failed to import\n")
+        self.assertIn(
+            "The pyomo.common.tests.dep_mod_except module "
+            "(an optional Pyomo dependency) failed to import",
+            log.getvalue())
 
     def test_importer(self):
         attempted_import = []
diff --git a/pyomo/contrib/pynumero/interfaces/tests/test_dynamic_model.py b/pyomo/contrib/pynumero/interfaces/tests/test_dynamic_model.py
--- a/pyomo/contrib/pynumero/interfaces/tests/test_dynamic_model.py
+++ b/pyomo/contrib/pynumero/interfaces/tests/test_dynamic_model.py
@@ -25,7 +25,7 @@
         "Pynumero needs the ASL extension to run CyIpoptSolver tests")
 
 from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import (
-    CyIpoptSolver, CyIpoptNLP, ipopt, ipopt_available,
+    CyIpoptSolver, CyIpoptNLP, cyipopt_available,
 )
 
 from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock
@@ -368,9 +368,12 @@ def max_outflow(m, t):
 
     return m2
 
+
 class TestGreyBoxModel(unittest.TestCase):
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(
+        not pyo.SolverFactory('ipopt').available(exception_flag=False),
+        "Ipopt needed to run tests with solve")
     def test_compare_evaluations(self):
         A1 = 5
         A2 = 10
@@ -471,7 +474,9 @@ def test_compare_evaluations(self):
         mex_nlp.evaluate_hessian_lag(out=mex_h)
         check_sparse_matrix_specific_order(self, m_h, m_x_order, m_x_order, mex_h, mex_x_order, mex_x_order, x1_x2_map, x1_x2_map)
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_solve(self):
         A1 = 5
         A2 = 10
diff --git a/pyomo/contrib/pynumero/interfaces/tests/test_external_grey_box_model.py b/pyomo/contrib/pynumero/interfaces/tests/test_external_grey_box_model.py
--- a/pyomo/contrib/pynumero/interfaces/tests/test_external_grey_box_model.py
+++ b/pyomo/contrib/pynumero/interfaces/tests/test_external_grey_box_model.py
@@ -24,7 +24,7 @@
         "Pynumero needs the ASL extension to run cyipopt tests")
 
 from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import (
-    ipopt, ipopt_available,
+    cyipopt_available,
 )
 
 from ..external_grey_box import ExternalGreyBoxModel, ExternalGreyBoxBlock
@@ -1217,7 +1217,8 @@ def _test_external_additional_constraints_vars(self, ex_model, hessian_support):
             with self.assertRaises(AttributeError):
                 h = pyomo_nlp.evaluate_hessian_lag()
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_external_greybox_solve(self):
         self._test_external_greybox_solve(ex_models.PressureDropTwoEqualitiesTwoOutputs(), False)
         self._test_external_greybox_solve(ex_models.PressureDropTwoEqualitiesTwoOutputsWithHessian(), True)
@@ -1423,7 +1424,8 @@ def test_scaling_pyomo_model_and_greybox(self):
         comparison_cs = np.asarray([3.1, 3.2, 4.1, 4.2, 1, 2.2], dtype=np.float64)
         check_vectors_specific_order(self, cs, c_order, comparison_cs, comparison_c_order)
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_external_greybox_solve_scaling(self):
         m = pyo.ConcreteModel()
         m.mu = pyo.Var(bounds=(0,None), initialize=1)
diff --git a/pyomo/contrib/pynumero/interfaces/tests/test_pyomo_grey_box_nlp.py b/pyomo/contrib/pynumero/interfaces/tests/test_pyomo_grey_box_nlp.py
--- a/pyomo/contrib/pynumero/interfaces/tests/test_pyomo_grey_box_nlp.py
+++ b/pyomo/contrib/pynumero/interfaces/tests/test_pyomo_grey_box_nlp.py
@@ -24,7 +24,7 @@
         "Pynumero needs the ASL extension to run cyipopt tests")
 
 from pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver import (
-    ipopt, ipopt_available,
+    cyipopt_available,
 )
 
 from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxBlock
@@ -1419,7 +1419,8 @@ def _test_external_additional_constraints_vars(self, ex_model, hessian_support):
             with self.assertRaises(NotImplementedError):
                 h = pyomo_nlp.evaluate_hessian_lag()
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_external_greybox_solve(self):
         self._test_external_greybox_solve(ex_models.PressureDropTwoEqualitiesTwoOutputs(), False)
         self._test_external_greybox_solve(ex_models.PressureDropTwoEqualitiesTwoOutputsWithHessian(), True)
@@ -1623,7 +1624,8 @@ def test_scaling_pyomo_model_and_greybox(self):
         comparison_cs = np.asarray([3.1, 3.2, 4.1, 4.2, 1, 2.2], dtype=np.float64)
         check_vectors_specific_order(self, cs, c_order, comparison_cs, comparison_c_order)
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_external_greybox_solve_scaling(self):
         m = pyo.ConcreteModel()
         m.mu = pyo.Var(bounds=(0,None), initialize=1)
@@ -1706,7 +1708,8 @@ def test_external_greybox_solve_scaling(self):
         self.assertIn('DenseVector "d scaling vector" with 1 elements:', solver_trace)
         self.assertIn('d scaling vector[    1]= 1.0000000000000000e+00', solver_trace) # pcon
 
-    @unittest.skipIf(not ipopt_available, "CyIpopt needed to run tests with solve")
+    @unittest.skipIf(not cyipopt_available,
+                     "CyIpopt needed to run tests with solve")
     def test_duals_after_solve(self):
         m = pyo.ConcreteModel()
         m.p = pyo.Var(initialize=1)
diff --git a/pyomo/contrib/pynumero/tests/test_cyipopt_examples.py b/pyomo/contrib/pynumero/tests/test_cyipopt_examples.py
--- a/pyomo/contrib/pynumero/tests/test_cyipopt_examples.py
+++ b/pyomo/contrib/pynumero/tests/test_cyipopt_examples.py
@@ -37,18 +37,17 @@
         "Pynumero needs the ASL extension to run CyIpopt tests")
 
 import pyomo.contrib.pynumero.algorithms.solvers.cyipopt_solver as cyipopt_solver
-if not cyipopt_solver.ipopt_available:
-    raise unittest.SkipTest("PyNumero needs CyIpopt installed to run CyIpopt tests")
+if not cyipopt_solver.cyipopt_available:
+    raise unittest.SkipTest(
+        "PyNumero needs CyIpopt installed to run CyIpopt tests")
 import cyipopt as cyipopt_core
 
 example_dir = os.path.join(this_file_dir(), '..', 'examples')
 
 class TestPyomoCyIpoptSolver(unittest.TestCase):
     def test_status_maps(self):
-        self.assertEqual(len(cyipopt_core.STATUS_MESSAGES),
-                         len(cyipopt_solver._cyipopt_status_enum))
-        self.assertEqual(len(cyipopt_core.STATUS_MESSAGES),
-                         len(cyipopt_solver._ipopt_term_cond))
+        # verify that all status messages from cyipopy can be cleanly
+        # mapped back to a Pyomo TerminationCondition
         for msg in cyipopt_core.STATUS_MESSAGES.values():
             self.assertIn(msg, cyipopt_solver._cyipopt_status_enum)
         for status in cyipopt_solver._cyipopt_status_enum.values():