Repository: Pyomo/pyomo

Issue:
core.base.symbolic.differentiate() does not support indexed Expressions
It looks like the current differentiate() implementation does not support indexed Expression objects. Turning y into a variable and adding a constraint works, but may not be desirable in some cases.

Simple example:
```
m = ConcreteModel()
m.s = Set(initialize=['A', 'B'])
m.x = Var(m.s, domain=NonNegativeReals)

def y_rule(m, s):
    return m.x[s] * 2
m.y = Expression(m.s, rule=y_rule)

@m.Constraint(m.s)
def c1(m, s):
    return 1 - m.y[s] ** 2 <= 0

jacs = differentiate(m.c1['A'].body, wrt_list=[m.x['A'], m.x['B']])
```

Stack Trace:
```
  File "test.py", line 22, in main
    jacs = differentiate(m.c1['A'].body, wrt_list=[m.x['A'], m.x['B']])
  File "/home/qichen/.solvers/pyomo/src/pyomo/pyomo/core/base/symbolic.py", line 122, in differentiate
    tmp_expr, locals=dict((str(x), x) for x in sympy_vars) )
  File "/home/qichen/.solvers/pyomo/local/lib/python2.7/site-packages/sympy/core/sympify.py", line 322, in sympify
    expr = parse_expr(a, local_dict=locals, transformations=transformations, evaluate=evaluate)
  File "/home/qichen/.solvers/pyomo/local/lib/python2.7/site-packages/sympy/parsing/sympy_parser.py", line 894, in parse_expr
    return eval_expr(code, local_dict, global_dict)
  File "/home/qichen/.solvers/pyomo/local/lib/python2.7/site-packages/sympy/parsing/sympy_parser.py", line 807, in eval_expr
    code, global_dict, local_dict)  # take local objects in preference
  File "<string>", line 1, in <module>
TypeError: 'Symbol' object has no attribute '__getitem__'
```


Discussion:

Resolving patch:
diff --git a/pyomo/core/base/expression.py b/pyomo/core/base/expression.py
--- a/pyomo/core/base/expression.py
+++ b/pyomo/core/base/expression.py
@@ -72,7 +72,7 @@ def _args_(self):
 
     @property
     def args(self):
-        yield self.expr
+        return (self.expr,)
 
     def nargs(self):
         return 1
diff --git a/pyomo/core/base/symbolic.py b/pyomo/core/base/symbolic.py
--- a/pyomo/core/base/symbolic.py
+++ b/pyomo/core/base/symbolic.py
@@ -8,13 +8,13 @@
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 
-from six import StringIO
+from six import StringIO, iterkeys
 import pyutilib.misc
 from pyomo import core
-from pyomo.core.expr import current as EXPR
-from pyomo.core.expr import native_types
 from pyomo.common import DeveloperError
+from pyomo.core.expr import current as EXPR, native_types
 from pyomo.core.expr.numvalue import value
+from pyomo.core.kernel.component_map import ComponentMap
 
 _sympy_available = True
 try:
@@ -65,6 +65,13 @@ def _nondifferentiable(*x):
         sympy.Tuple: lambda *x: x,
     }
 
+    _pyomo_operator_map = {
+        EXPR.SumExpression: sympy.Add,
+        EXPR.ProductExpression: sympy.Mul,
+        EXPR.NPV_ProductExpression: sympy.Mul,
+        EXPR.MonomialTermExpression: sympy.Mul,
+    }
+
     _functionMap = {
         'exp': sympy.exp,
         'log': sympy.log,
@@ -85,7 +92,7 @@ def _nondifferentiable(*x):
         'floor': sympy.floor,
         'sqrt': sympy.sqrt,
     }
-except ImportError: #pragma:nocover
+except ImportError:
     _sympy_available = False
 
 # A "public" attribute indicating that differentiate() can be called
@@ -97,6 +104,26 @@ class NondifferentiableError(ValueError):
     """A Pyomo-specific ValueError raised for non-differentiable expressions"""
     pass
 
+class PyomoSympyBimap(object):
+    def __init__(self):
+        self.pyomo2sympy = ComponentMap()
+        self.sympy2pyomo = {}
+        self.i = 0
+
+    def getPyomoSymbol(self, sympy_object, default=None):
+        return self.sympy2pyomo.get(sympy_object, default)
+
+    def getSympySymbol(self, pyomo_object):
+        if pyomo_object in self.pyomo2sympy:
+            return self.pyomo2sympy[pyomo_object]
+        sympy_obj = sympy.Symbol("x%d" % self.i)
+        self.i += 1
+        self.pyomo2sympy[pyomo_object] = sympy_obj
+        self.sympy2pyomo[sympy_obj] = pyomo_object
+        return sympy_obj
+
+    def sympyVars(self):
+        return iterkeys(self.sympy2pyomo)
 
 def differentiate(expr, wrt=None, wrt_list=None):
     """Return derivative of expression.
@@ -116,12 +143,23 @@ def differentiate(expr, wrt=None, wrt_list=None):
     """
     if not _sympy_available:
         raise RuntimeError(
-            "The sympy module is not available.  "
+            "The sympy module is not available.\n\t"
             "Cannot perform automatic symbolic differentiation.")
     if not (( wrt is None ) ^ ( wrt_list is None )):
         raise ValueError(
             "differentiate(): Must specify exactly one of wrt and wrt_list")
     #
+    # Convert the Pyomo expression to a sympy expression
+    #
+    objectMap, sympy_expr = sympify_expression(expr)
+    #
+    # The partial_derivs dict holds intermediate sympy expressions that
+    # we can re-use.  We will prepopulate it with None for all vars that
+    # appear in the expression (so that we can detect wrt combinations
+    # that are, by definition, 0)
+    #
+    partial_derivs = dict((x,None) for x in objectMap.sympyVars())
+    #
     # Setup the WRT list
     #
     if wrt is not None:
@@ -130,45 +168,42 @@ def differentiate(expr, wrt=None, wrt_list=None):
         # Copy the list because we will normalize things in place below
         wrt_list = list(wrt_list)
     #
-    # Setup mapping dictionaries
-    #
-    pyomo_vars = list(EXPR.identify_variables(expr))
-    pyomo_vars = sorted(pyomo_vars, key=lambda x: str(x))
-    sympy_vars = [sympy.var('x%s'% i) for i in range(len(pyomo_vars))]
-    sympy2pyomo = dict( zip(sympy_vars, pyomo_vars) )
-    pyomo2sympy = dict( (id(pyomo_vars[i]), sympy_vars[i])
-                         for i in range(len(pyomo_vars)) )
+    # Convert WRT vars into sympy vars
     #
-    # Process WRT information
-    #
-    ans = []
+    ans = [None]*len(wrt_list)
     for i, target in enumerate(wrt_list):
         if target.__class__ is not tuple:
-            wrt_list[i] = target = (target,)
-        mismatch_target = False
-        for var in target:
-            if id(var) not in pyomo2sympy:
-                mismatch_target = True
+            target = (target,)
+        wrt_list[i] = tuple(objectMap.getSympySymbol(x) for x in target)
+        for x in wrt_list[i]:
+            if x not in partial_derivs:
+                ans[i] = 0.
                 break
-        wrt_list[i] = tuple( pyomo2sympy.get(id(var),None) for var in target )
-        ans.append(0 if mismatch_target else None)
-    #
-    # If there is nothing to do, do nothing
-    #
-    if all(i is not None for i in ans):
-        return ans if wrt is None else ans[0]
     #
-    # Create sympy expression
+    # We assume that users will not request duplicate derivatives.  We
+    # will only cache up to the next-to last partial, and if a user
+    # requests the exact same derivative twice, then we will just
+    # re-calculate it.
     #
-    sympy_expr = sympify_expression(expr, sympy2pyomo, pyomo2sympy)
+    last_partial_idx = max(len(x) for x in wrt_list) - 1
     #
-    # Differentiate for each WRT variable, and map the
-    # result back into a Pyomo expression tree.
+    # Calculate all the derivatives
     #
     for i, target in enumerate(wrt_list):
-        if ans[i] is None:
-            sympy_ans = sympy_expr.diff(*target)
-            ans[i] = _map_sympy2pyomo(sympy_ans, sympy2pyomo)
+        if ans[i] is not None:
+            continue
+        part = sympy_expr
+        for j, wrt_var in enumerate(target):
+            if j == last_partial_idx:
+                part = sympy.diff(part, wrt_var)
+            else:
+                partial_target = target[:j+1]
+                if partial_target in partial_derivs:
+                    part = partial_derivs[partial_target]
+                else:
+                    part = sympy.diff(part, wrt_var)
+                    partial_derivs[partial_target] = part
+        ans[i] = sympy2pyomo_expression(part, objectMap)
     #
     # Return the answer
     #
@@ -179,71 +214,52 @@ def differentiate(expr, wrt=None, wrt_list=None):
 # sympify_expression
 # =====================================================
 
-class SympifyVisitor(EXPR.ExpressionValueVisitor):
+class Pyomo2SympyVisitor(EXPR.StreamBasedExpressionVisitor):
 
-    def __init__(self, native_or_sympy_types, pyomo2sympy):
-        self.native_or_sympy_types = native_or_sympy_types
-        self.pyomo2sympy = pyomo2sympy
+    def __init__(self, object_map):
+        super(Pyomo2SympyVisitor, self).__init__()
+        self.object_map = object_map
 
-    def visit(self, node, values):
+    def exitNode(self, node, values):
         if node.__class__ is EXPR.UnaryFunctionExpression:
             return _functionMap[node._name](values[0])
-        else:
+        _op = _pyomo_operator_map.get(node.__class__, None)
+        if _op is None:
             return node._apply_operation(values)
+        else:
+            return _op(*tuple(values))
 
-    def visiting_potential_leaf(self, node):
+    def beforeChild(self, node, child):
         #
         # Don't replace native or sympy types
         #
-        if type(node) in self.native_or_sympy_types:
-            return True, node
+        if type(child) in native_types:
+            return False, child
         #
-        # Replace pyomo variables with sympy variables
+        # We will descend into all expressions...
         #
-        if id(node) in self.pyomo2sympy:
-            return True, self.pyomo2sympy[id(node)]
+        if child.is_expression_type():
+            return True, None
         #
-        # Replace constants
+        # Replace pyomo variables with sympy variables
         #
-        if not node.is_potentially_variable():
-            return True, value(node)
+        if child.is_potentially_variable():
+            return False, self.object_map.getSympySymbol(child)
         #
-        # Don't replace anything else
+        # Everything else is a constant...
         #
-        return False, None
+        return False, value(child)
 
-    def finalize(self, ans):
-        return ans
-
-def sympify_expression(expr, sympySymbols, pyomo2sympy):
-    #
-    # Handle simple cases
-    #
-    if expr.__class__ in native_types:
-        return expr
-    if not expr.is_expression_type():
-        if id(expr) in pyomo2sympy:
-            return pyomo2sympy[id(expr)]
-        return expr
-    #
-    # Create the visitor and call it.
-    #
-    native_or_sympy_types = set(native_types)
-    native_or_sympy_types.add( type(list(sympySymbols)[0]) )
-    visitor = SympifyVisitor(native_or_sympy_types, pyomo2sympy)
-    return visitor.dfs_postorder_stack(expr)
-
-
-# =====================================================
-# _map_sympy2pyomo
-# =====================================================
+class Sympy2PyomoVisitor(EXPR.StreamBasedExpressionVisitor):
 
-class Sympy2PyomoVisitor(pyutilib.misc.ValueVisitor):
+    def __init__(self, object_map):
+        super(Sympy2PyomoVisitor, self).__init__()
+        self.object_map = object_map
 
-    def __init__(self, sympy2pyomo):
-        self.sympy2pyomo = sympy2pyomo
+    def enterNode(self, node):
+        return (node._args, [])
 
-    def visit(self, node, values):
+    def exitNode(self, node, values):
         """ Visit nodes that have been expanded """
         _sympyOp = node
         _op = _operatorMap.get( type(_sympyOp), None )
@@ -253,30 +269,31 @@ def visit(self, node, values):
                 "map" % type(_sympyOp) )
         return _op(*tuple(values))
 
-    def visiting_potential_leaf(self, node):
-        """
-        Visiting a potential leaf.
+    def beforeChild(self, node, child):
+        if not child._args:
+            item = self.object_map.getPyomoSymbol(child, None)
+            if item is None:
+                item = float(child.evalf())
+            return False, item
+        return True, None
 
-        Return True if the node is not expanded.
-        """
-        if not node._args:
-            if node in self.sympy2pyomo:
-                return True, self.sympy2pyomo[node]
-            else:
-                return True, float(node.evalf())
-        return False, None
+def sympify_expression(expr):
+    """Convert a Pyomo expression to a Sympy expression"""
+    #
+    # Create the visitor and call it.
+    #
+    object_map = PyomoSympyBimap()
+    visitor = Pyomo2SympyVisitor(object_map)
+    is_expr, ans = visitor.beforeChild(None, expr)
+    if not is_expr:
+        return object_map, ans
 
-    def children(self, node):
-        return list(node._args)
+    return object_map, visitor.walk_expression(expr)
 
-    def finalize(self, ans):
-        return ans
 
-def _map_sympy2pyomo(expr, sympy2pyomo):
-    if not expr._args:
-        if expr in sympy2pyomo:
-            return sympy2pyomo[expr]
-        else:
-            return float(expr.evalf())
-    visitor = Sympy2PyomoVisitor(sympy2pyomo)
-    return visitor.dfs_postorder_stack(expr)
+def sympy2pyomo_expression(expr, object_map):
+    visitor = Sympy2PyomoVisitor(object_map)
+    is_expr, ans = visitor.beforeChild(None, expr)
+    if not is_expr:
+        return ans
+    return visitor.walk_expression(expr)
diff --git a/pyomo/core/expr/expr_pyomo5.py b/pyomo/core/expr/expr_pyomo5.py
--- a/pyomo/core/expr/expr_pyomo5.py
+++ b/pyomo/core/expr/expr_pyomo5.py
@@ -398,7 +398,7 @@ def walk_expression(self, expr):
                         # We are aborting processing of this child node.
                         # Tell this node to accept the child result and
                         # we will move along
-                        if self.acceptChildResult:
+                        if self.acceptChildResult is not None:
                             data = self.acceptChildResult(
                                 node, data, child_result)
                         elif data is not None:


Test patch:
diff --git a/pyomo/core/tests/unit/test_symbolic.py b/pyomo/core/tests/unit/test_symbolic.py
--- a/pyomo/core/tests/unit/test_symbolic.py
+++ b/pyomo/core/tests/unit/test_symbolic.py
@@ -14,8 +14,8 @@
 from pyomo.common import DeveloperError
 from pyomo.core import *
 from pyomo.core.base.symbolic import (
-    differentiate, NondifferentiableError,
-    _sympy_available, _map_sympy2pyomo,
+    differentiate, NondifferentiableError, PyomoSympyBimap,
+    _sympy_available, sympy2pyomo_expression,
 )
 
 def s(e):
@@ -244,6 +244,82 @@ def test_sqrt_function(self):
         self.assertTrue(e.is_expression_type())
         self.assertEqual(s(e), s(0.5 * m.x**-0.5))
 
+    def test_param(self):
+        m = ConcreteModel()
+        m.x = Var()
+        m.p = Param(mutable=True, initialize=5)
+
+        e = differentiate(m.p*m.x, wrt=m.x)
+        self.assertIs(type(e), float)
+        self.assertEqual(e, 5.0)
+
+    def test_Expression_component(self):
+        m = ConcreteModel()
+        m.s = Set(initialize=['A', 'B'])
+        m.x = Var(m.s, domain=NonNegativeReals)
+
+        def y_rule(m, s):
+            return m.x[s] * 2
+        m.y = Expression(m.s, rule=y_rule)
+
+        expr = 1 - m.y['A'] ** 2
+        jacs = differentiate(expr, wrt_list=[m.x['A'], m.x['B']])
+        self.assertEqual(str(jacs[0]), "-8.0*x[A]")
+        self.assertEqual(str(jacs[1]), "0.0")
+
+        expr = 1 - m.y['B'] ** 2
+        jacs = differentiate(expr, wrt_list=[m.x['A'], m.x['B']])
+        self.assertEqual(str(jacs[0]), "0.0")
+        self.assertEqual(str(jacs[1]), "-8.0*x[B]")
+
+    def test_jacobian(self):
+        m = ConcreteModel()
+        m.I = RangeSet(4)
+        m.x = Var(m.I)
+
+        idxMap = {}
+        jacs = []
+        for i in m.I:
+            idxMap[i] = len(jacs)
+            jacs.append(m.x[i])
+
+        expr = m.x[1]+m.x[2]*m.x[3]**2
+        ans = differentiate(expr, wrt_list=jacs)
+
+        self.assertEqual(len(ans), len(m.I))
+        self.assertEqual(str(ans[0]), "1.0")
+        self.assertEqual(str(ans[1]), "x[3]**2.0")
+        self.assertEqual(str(ans[2]), "2.0*x[2]*x[3]")
+        # 0 calculated by bypassing sympy
+        self.assertEqual(str(ans[3]), "0.0")
+
+    def test_hessian(self):
+        m = ConcreteModel()
+        m.I = RangeSet(4)
+        m.x = Var(m.I)
+
+        idxMap = {}
+        hessian = []
+        for i in m.I:
+            for j in m.I:
+                idxMap[i,j] = len(hessian)
+                hessian.append((m.x[i], m.x[j]))
+
+        expr = m.x[1]+m.x[2]*m.x[3]**2
+        ans = differentiate(expr, wrt_list=hessian)
+
+        self.assertEqual(len(ans), len(m.I)**2)
+        for i in m.I:
+            for j in m.I:
+                self.assertEqual(str(ans[idxMap[i,j]]), str(ans[idxMap[j,i]]))
+        # 0 calculated by sympy
+        self.assertEqual(str(ans[idxMap[1,1]]), "0.0")
+        self.assertEqual(str(ans[idxMap[2,2]]), "0.0")
+        self.assertEqual(str(ans[idxMap[3,3]]), "2.0*x[2]")
+        # 0 calculated by bypassing sympy
+        self.assertEqual(str(ans[idxMap[4,4]]), "0.0")
+        self.assertEqual(str(ans[idxMap[2,3]]), "2.0*x[3]")
+
     def test_nondifferentiable(self):
         m = ConcreteModel()
         m.foo = Var()
@@ -269,14 +345,23 @@ def test_errors(self):
             "Must specify exactly one of wrt and wrt_list",
             differentiate, m.x, wrt=m.x, wrt_list=[m.x])
 
-        x = pyomo.core.base.symbolic.sympy.Symbol('x')
+        obj_map = PyomoSympyBimap()
         class bogus(object):
             def __init__(self):
-                self._args = (x,)
+                self._args = (obj_map.getSympySymbol(m.x),)
         self.assertRaisesRegexp(
             DeveloperError,
             "sympy expression .* not found in the operator map",
-            _map_sympy2pyomo, bogus(), {x:m.x})
+            sympy2pyomo_expression, bogus(), obj_map)
+
+
+class SymbolicDerivatives_importTest(unittest.TestCase):
+    def test_sympy_avail_flag(self):
+        if _sympy_available:
+            import sympy
+        else:
+            with self.assertRaises(ImportError):
+                import sympy
 
 if __name__ == "__main__":
     unittest.main()