Repository: Pyomo/pyomo

Issue:
Bug when using solve_strongly_connected_components
## Summary
I have been attempting to apply the `solve_strongly_connected_components` method to the IDAES model libraries and have encountered a couple of bugs the underlying Pyomo code. The first issue that comes up appears to be a missing import in `pyomo/core/base/external.py`, however fixing this unearths a deeper issue.


### Steps to reproduce the issue

The best replication I have is to run the IDAES test suite using the following branch: https://github.com/andrewlee94/idaes-pse/tree/bt_solver

One test that reproduces this error is:

`pytest idaes/models/properties/modular_properties/examples/tests/test_CO2_bmimPF6_PR.py::TestFlashIntegration::test_initialize`

### Error Message

The first bug lies in `external.py`, where it appears that the `value` method needs to be imported from `pyomo.core.expr.numvalue`

```
../../pyomo/pyomo/pyomo/core/base/external.py:214: in evaluate
    args_ = [arg if arg.__class__ in native_types else value(arg)

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
>   args_ = [arg if arg.__class__ in native_types else value(arg) for arg in args]
E   NameError: name 'value' is not defined

../../pyomo/pyomo/pyomo/core/base/external.py:214: NameError
```

However, fixing this results in a second error from deeper in the code:

```
idaes/core/base/unit_model.py:527: in initialize
    return solve_strongly_connected_components(
../../pyomo/pyomo/pyomo/contrib/incidence_analysis/util.py:147: in solve_strongly_connected_components
    results = calculate_variable_from_constraint(
../../pyomo/pyomo/pyomo/util/calc_var_value.py:158: in calculate_variable_from_constraint
    expr_deriv = differentiate(expr, wrt=variable,
../../pyomo/pyomo/pyomo/core/expr/calculus/derivatives.py:116: in differentiate
    res = sympy_diff(expr=expr, wrt=wrt, wrt_list=wrt_list)
../../pyomo/pyomo/pyomo/core/expr/calculus/diff_with_sympy.py:47: in differentiate
    objectMap, sympy_expr = sympyify_expression(expr)
../../pyomo/pyomo/pyomo/core/expr/sympy_tools.py:215: in sympyify_expression
    return object_map, visitor.walk_expression(expr)
../../pyomo/pyomo/pyomo/core/expr/visitor.py:256: in walk_expression
    result = self._process_node(root, RECURSION_LIMIT)
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:455: in _process_node_bx
    data.append(self._process_node(child, recursion_limit))
../../pyomo/pyomo/pyomo/core/expr/visitor.py:466: in _process_node_bx
    return self.exitNode(node, data)
../../pyomo/pyomo/pyomo/core/expr/sympy_tools.py:152: in exitNode
    return node._apply_operation(values)
../../pyomo/pyomo/pyomo/core/expr/numeric_expr.py:650: in _apply_operation
    return self._fcn.evaluate( result )
../../pyomo/pyomo/pyomo/core/base/external.py:216: in evaluate
    return self._evaluate(args_, None, 0)[0]
../../pyomo/pyomo/pyomo/core/base/external.py:360: in _evaluate
    arglist = _ARGLIST(args, fgh, fixed)
../../pyomo/pyomo/pyomo/core/base/external.py:652: in __init__
    self.ra = (c_double*nr)(*_reals)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = 0.120272355044943*x6*(0.6468651916804*x0*x5/x1 + 0.6468651916804*x2*x3/x4)/x7 - 1.0

    def __float__(self):
        # Don't bother testing if it's a number; if it's not this is going
        # to fail, and if it is we still need to check that it evalf'ed to
        # a number.
        result = self.evalf()
        if result.is_Number:
            return float(result)
        if result.is_number and result.as_real_imag()[1]:
            raise TypeError("Cannot convert complex to float")

>       raise TypeError("Cannot convert expression to float")

E       TypeError: Cannot convert expression to float

../../anaconda3/lib/python3.9/site-packages/sympy/core/expr.py:345: TypeError
```



Discussion:
Looking a bit deeper into the stack trace, it appears to me that the issue is arising when the Newton solver in `calculate_variable_from_constraint` attempts to get the derivative of a constraint w.r.t. to a variable in some situations.

Resolving patch:
diff --git a/pyomo/util/calc_var_value.py b/pyomo/util/calc_var_value.py
--- a/pyomo/util/calc_var_value.py
+++ b/pyomo/util/calc_var_value.py
@@ -18,7 +18,8 @@
 
 def calculate_variable_from_constraint(variable, constraint,
                                        eps=1e-8, iterlim=1000,
-                                       linesearch=True, alpha_min=1e-8):
+                                       linesearch=True, alpha_min=1e-8,
+                                       diff_mode=differentiate.Modes.sympy):
     """Calculate the variable value given a specified equality constraint
 
     This function calculates the value of the specified variable
@@ -55,6 +56,8 @@ def calculate_variable_from_constraint(variable, constraint,
         [default=True]
     alpha_min: `float`
         The minimum fractional step to use in the linesearch [default=1e-8].
+    diff_mode: `pyomo.core.expr.calculus.derivatives.Modes`
+        The mode of differentiation
 
     Returns:
     --------
@@ -155,13 +158,21 @@ def calculate_variable_from_constraint(variable, constraint,
     # restore initial value
     variable.set_value(orig_initial_value, skip_validation=True)
     expr = body - upper
-    expr_deriv = differentiate(expr, wrt=variable,
-                               mode=differentiate.Modes.sympy)
 
-    if type(expr_deriv) in native_numeric_types and expr_deriv == 0:
-        raise ValueError("Variable derivative == 0, cannot solve for variable")
+    expr_deriv = None
+    if diff_mode in {differentiate.Modes.sympy, differentiate.Modes.reverse_symbolic}:
+        expr_deriv = differentiate(expr, wrt=variable,
+                                   mode=diff_mode)
 
-    if abs(value(expr_deriv)) < 1e-12:
+        if type(expr_deriv) in native_numeric_types and expr_deriv == 0:
+            raise ValueError("Variable derivative == 0, cannot solve for variable")
+
+    if expr_deriv is None:
+        fp0 = differentiate(expr, wrt=variable, mode=diff_mode)
+    else:
+        fp0 = value(expr_deriv)
+
+    if abs(value(fp0)) < 1e-12:
         raise RuntimeError(
             'Initial value for variable results in a derivative value that is '
             'very close to zero.\n\tPlease provide a different initial guess.')
@@ -190,7 +201,12 @@ def calculate_variable_from_constraint(variable, constraint,
                 "expression.\n\tPlease provide a different initial guess "
                 "or enable the linesearch if you have not.")
             raise
-        fpk = value(expr_deriv)
+
+        if expr_deriv is None:
+            fpk = differentiate(expr, wrt=variable, mode=diff_mode)
+        else:
+            fpk = value(expr_deriv)
+
         if abs(fpk) < 1e-12:
             raise RuntimeError(
                 "Newton's method encountered a derivative that was too "


Test patch:
diff --git a/pyomo/util/tests/test_calc_var_value.py b/pyomo/util/tests/test_calc_var_value.py
--- a/pyomo/util/tests/test_calc_var_value.py
+++ b/pyomo/util/tests/test_calc_var_value.py
@@ -21,6 +21,13 @@
 )
 from pyomo.util.calc_var_value import calculate_variable_from_constraint
 from pyomo.core.expr.calculus.diff_with_sympy import differentiate_available
+from pyomo.core.expr.calculus.derivatives import differentiate
+
+
+all_diff_modes = [differentiate.Modes.sympy,
+                  differentiate.Modes.reverse_symbolic,
+                  differentiate.Modes.reverse_numeric]
+
 
 class Test_calc_var(unittest.TestCase):
     def test_initialize_value(self):
@@ -29,39 +36,46 @@ def test_initialize_value(self):
         m.y = Var(initialize=0)
         m.c = Constraint(expr=m.x == 5)
 
-        m.x.set_value(None)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setlb(3)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setlb(-10)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setub(10)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setlb(3)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setlb(None)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
-        m.x.set_value(None)
         m.x.setub(-10)
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 5)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 5)
 
         m.lt = Constraint(expr=m.x <= m.y)
         with self.assertRaisesRegex(
@@ -73,26 +87,33 @@ def test_linear(self):
         m.x = Var()
         m.c = Constraint(expr=5*m.x == 10)
 
-        calculate_variable_from_constraint(m.x, m.c)
-        self.assertEqual(value(m.x), 2)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
+            self.assertEqual(value(m.x), 2)
 
     def test_constraint_as_tuple(self):
         m = ConcreteModel()
         m.x = Var()
         m.p = Param(initialize=15, mutable=True)
 
-        calculate_variable_from_constraint(m.x, 5*m.x == 5)
-        self.assertEqual(value(m.x), 1)
-        calculate_variable_from_constraint(m.x, (5*m.x, 10))
-        self.assertEqual(value(m.x), 2)
-        calculate_variable_from_constraint(m.x, (15, 5*m.x, m.p))
-        self.assertEqual(value(m.x), 3)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, 5*m.x == 5, diff_mode=mode)
+            self.assertEqual(value(m.x), 1)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, (5*m.x, 10), diff_mode=mode)
+            self.assertEqual(value(m.x), 2)
+        for mode in all_diff_modes:
+            m.x.set_value(None)
+            calculate_variable_from_constraint(m.x, (15, 5*m.x, m.p), diff_mode=mode)
+            self.assertEqual(value(m.x), 3)
         with self.assertRaisesRegex(
                 ValueError, "Constraint 'tuple' is a Ranged Inequality "
                 "with a variable upper bound."):
             calculate_variable_from_constraint(m.x, (15, 5*m.x, m.x))
 
-
     @unittest.skipIf(not differentiate_available, "this test requires sympy")
     def test_nonlinear(self):
         m = ConcreteModel()
@@ -100,62 +121,90 @@ def test_nonlinear(self):
         m.y = Var(initialize=0)
 
         m.c = Constraint(expr=m.x**2 == 16)
-        m.x.set_value(1.0) # set an initial value
-        calculate_variable_from_constraint(m.x, m.c, linesearch=False)
-        self.assertAlmostEqual(value(m.x), 4)
+        for mode in all_diff_modes:
+            m.x.set_value(1.0) # set an initial value
+            calculate_variable_from_constraint(
+                m.x, m.c, linesearch=False, diff_mode=mode
+            )
+            self.assertEqual(value(m.x), 4)
 
         # test that infeasible constraint throws error
         m.d = Constraint(expr=m.x**2 == -1)
-        m.x.set_value(1.25) # set the initial value
-        with self.assertRaisesRegex(
-               RuntimeError, r'Iteration limit \(10\) reached'):
-           calculate_variable_from_constraint(
-               m.x, m.d, iterlim=10, linesearch=False)
+        for mode in all_diff_modes:
+            m.x.set_value(1.25) # set the initial value
+            with self.assertRaisesRegex(
+                   RuntimeError, r'Iteration limit \(10\) reached'):
+               calculate_variable_from_constraint(
+                   m.x, m.d, iterlim=10, linesearch=False, diff_mode=mode
+               )
 
         # same problem should throw a linesearch error if linesearch is on
-        m.x.set_value(1.25) # set the initial value
-        with self.assertRaisesRegex(
-                RuntimeError, "Linesearch iteration limit reached"):
-           calculate_variable_from_constraint(
-               m.x, m.d, iterlim=10, linesearch=True)
+        for mode in all_diff_modes:
+            m.x.set_value(1.25) # set the initial value
+            with self.assertRaisesRegex(
+                    RuntimeError, "Linesearch iteration limit reached"):
+               calculate_variable_from_constraint(
+                   m.x, m.d, iterlim=10, linesearch=True, diff_mode=mode
+               )
 
         # same problem should raise an error if initialized at 0
-        m.x = 0
-        with self.assertRaisesRegex(
-                RuntimeError, "Initial value for variable results in a "
-                "derivative value that is very close to zero."):
-            calculate_variable_from_constraint(m.x, m.c)
+        for mode in all_diff_modes:
+            m.x = 0
+            with self.assertRaisesRegex(
+                    RuntimeError, "Initial value for variable results in a "
+                                  "derivative value that is very close to zero."):
+                calculate_variable_from_constraint(m.x, m.c, diff_mode=mode)
 
         # same problem should raise a value error if we are asked to
         # solve for a variable that is not present
-        with self.assertRaisesRegex(
-                ValueError, "Variable derivative == 0"):
-            calculate_variable_from_constraint(m.y, m.c)
-
+        for mode in all_diff_modes:
+            if mode == differentiate.Modes.reverse_numeric:
+                # numeric differentiation should not be used to check if a
+                # derivative is always zero
+                with self.assertRaisesRegex(
+                        RuntimeError, "Initial value for variable results in a "
+                                      "derivative value that is very close to zero."):
+                    calculate_variable_from_constraint(m.y, m.c, diff_mode=mode)
+            else:
+                with self.assertRaisesRegex(
+                        ValueError, "Variable derivative == 0"):
+                    calculate_variable_from_constraint(m.y, m.c, diff_mode=mode)
 
         # should succeed with or without a linesearch
         m.e = Constraint(expr=(m.x - 2.0)**2 - 1 == 0)
-        m.x.set_value(3.1)
-        calculate_variable_from_constraint(m.x, m.e, linesearch=False)
-        self.assertAlmostEqual(value(m.x), 3)
+        for mode in all_diff_modes:
+            m.x.set_value(3.1)
+            calculate_variable_from_constraint(
+                m.x, m.e, linesearch=False, diff_mode=mode
+            )
+            self.assertAlmostEqual(value(m.x), 3)
 
-        m.x.set_value(3.1)
-        calculate_variable_from_constraint(m.x, m.e, linesearch=True)
-        self.assertAlmostEqual(value(m.x), 3)
+        for mode in all_diff_modes:
+            m.x.set_value(3.1)
+            calculate_variable_from_constraint(
+                m.x, m.e, linesearch=True, diff_mode=mode
+            )
+            self.assertAlmostEqual(value(m.x), 3)
 
 
         # we expect this to succeed with the linesearch
         m.f = Constraint(expr=1.0/(1.0+exp(-m.x))-0.5 == 0)
-        m.x.set_value(3.0)
-        calculate_variable_from_constraint(m.x, m.f, linesearch=True)
-        self.assertAlmostEqual(value(m.x), 0)
+        for mode in all_diff_modes:
+            m.x.set_value(3.0)
+            calculate_variable_from_constraint(
+                m.x, m.f, linesearch=True, diff_mode=mode
+            )
+            self.assertAlmostEqual(value(m.x), 0)
 
         # we expect this to fail without a linesearch
-        m.x.set_value(3.0)
-        with self.assertRaisesRegex(
-                RuntimeError, "Newton's method encountered a derivative "
-                "that was too close to zero"):
-            calculate_variable_from_constraint(m.x, m.f, linesearch=False)
+        for mode in all_diff_modes:
+            m.x.set_value(3.0)
+            with self.assertRaisesRegex(
+                    RuntimeError, "Newton's method encountered a derivative "
+                    "that was too close to zero"):
+                calculate_variable_from_constraint(
+                    m.x, m.f, linesearch=False, diff_mode=mode
+                )
 
         # Calculate the bubble point of Benzene.  THe first step
         # computed by calculate_variable_from_constraint will make the
@@ -240,20 +289,24 @@ def test_bound_violation(self):
         m.c1 = Constraint(expr=m.v1 == 0)
 
         # Calculate value of v1 using constraint c1
-        calculate_variable_from_constraint(m.v1, m.c1)
-        self.assertEqual(value(m.v1), 0)
+        for mode in all_diff_modes:
+            m.v1.set_value(None)
+            calculate_variable_from_constraint(m.v1, m.c1, diff_mode=mode)
+            self.assertEqual(value(m.v1), 0)
 
         # Calculate value of v1 using a scaled constraint c2
         m.c2 = Constraint(expr=m.v1*10 == 0)
-        m.v1.set_value(1)
-        calculate_variable_from_constraint(m.v1, m.c2)
-        self.assertEqual(value(m.v1), 0)
+        for mode in all_diff_modes:
+            m.v1.set_value(1)
+            calculate_variable_from_constraint(m.v1, m.c2, diff_mode=mode)
+            self.assertEqual(value(m.v1), 0)
 
         # Test linear solution falling outside bounds
         m.c3 = Constraint(expr=m.v1*10 == -1)
-        m.v1.set_value(1)
-        calculate_variable_from_constraint(m.v1, m.c3)
-        self.assertEqual(value(m.v1), -0.1)
+        for mode in all_diff_modes:
+            m.v1.set_value(1)
+            calculate_variable_from_constraint(m.v1, m.c3, diff_mode=mode)
+            self.assertEqual(value(m.v1), -0.1)
 
     @unittest.skipUnless(differentiate_available, "this test requires sympy")
     def test_nonlinear_bound_violation(self):
@@ -263,9 +316,10 @@ def test_nonlinear_bound_violation(self):
 
         # Test nonlinear solution falling outside bounds
         m.c4 = Constraint(expr=m.v1**3 == -8)
-        m.v1.set_value(1)
-        calculate_variable_from_constraint(m.v1, m.c4)
-        self.assertEqual(value(m.v1), -2)
+        for mode in all_diff_modes:
+            m.v1.set_value(1)
+            calculate_variable_from_constraint(m.v1, m.c4, diff_mode=mode)
+            self.assertEqual(value(m.v1), -2)
 
     def test_warn_final_value_linear(self):
         m = ConcreteModel()