Repository: Pyomo/pyomo

Issue:
Access to derivative information from AMPL external functions.
This is IDAES related.  

For testing purposes, I would like to access derivative information from AMPLExternalFunction calls.  It's a lot easier to write tests in Python than C and the IDAES infrastructure is geared toward Python. It looks like in the code derivatives are returned through a local variable arglist in the evaluate function.  I'd like to keep a reference around so after the function evaluation I can pull out the derivative information.

Preliminarily I went here: https://github.com/Pyomo/pyomo/blob/master/pyomo/core/base/external.py#L120, and just changed arglist to self.arglist.  I think that provides what I'm looking for, but I don't know if that's the best way to go.
Access to derivative information from AMPL external functions.
This is IDAES related.  

For testing purposes, I would like to access derivative information from AMPLExternalFunction calls.  It's a lot easier to write tests in Python than C and the IDAES infrastructure is geared toward Python. It looks like in the code derivatives are returned through a local variable arglist in the evaluate function.  I'd like to keep a reference around so after the function evaluation I can pull out the derivative information.

Preliminarily I went here: https://github.com/Pyomo/pyomo/blob/master/pyomo/core/base/external.py#L120, and just changed arglist to self.arglist.  I think that provides what I'm looking for, but I don't know if that's the best way to go.


Discussion:

Resolving patch:
diff --git a/pyomo/core/base/external.py b/pyomo/core/base/external.py
--- a/pyomo/core/base/external.py
+++ b/pyomo/core/base/external.py
@@ -101,7 +101,7 @@ def __init__(self, *args, **kwds):
         self._so = None
         ExternalFunction.__init__(self, *args, **kwds)
 
-    def evaluate(self, args):
+    def _evaluate(self, args, fgh, fixed):
         if self._so is None:
             self.load_library()
         if self._function not in self._known_functions:
@@ -117,9 +117,22 @@ def evaluate(self, args):
                 args_.append(arg.value)
             else:
                 args_.append(arg)
-        arglist = _ARGLIST( *args_ )
+        arglist = _ARGLIST( args_, fgh, fixed )
         fcn = self._known_functions[self._function][0]
-        return fcn(byref(arglist))
+        f = fcn(byref(arglist))
+        return f, arglist
+
+    def evaluate(self, args):
+        return self._evaluate(args, 0, None)[0]
+
+    def evaluate_fgh(self, args, fixed=None):
+        if fixed is None:
+            fixed = tuple( arg.__class__ in native_types or arg.is_fixed()
+                           for arg in args )
+        f, arglist = self._evaluate(args, 2, fixed)
+        g = [arglist.derivs[i] for i in range(len(args))]
+        h = [arglist.hes[i] for i in range(len(args)**2)]
+        return f, g, h
 
     def load_library(self):
         try:
@@ -251,23 +264,27 @@ class _ARGLIST(Structure):
         ('nsout', c_int), # number of symbolic OUT and INOUT args
         ]
 
-    def __init__(self, *args):
+    def __init__(self, args, fgh=0, fixed=None):
         super(_ARGLIST, self).__init__()
         N = len(args)
         self.n = self.nr = N
         self.at = (c_int*N)()
         self.ra = (c_double*N)()
         self.sa = None
-        self.derivs = (c_double*N)()
-        self.hes = (c_double*(N*N))()
+        if fgh > 0:
+            self.derivs = (c_double*N)(0.)
+        if fgh > 1:
+            self.hes = (c_double*(N*N))(0.)
 
         for i,v in enumerate(args):
             self.at[i] = i
             self.ra[i] = v
-            self.derivs[i] = 0.
-        for i in six.moves.xrange(N*N):
-            self.hes[i] = 0.
 
+        if fixed:
+            self.dig = (c_byte*N)(0)
+            for i,v in enumerate(fixed):
+                if v:
+                    self.dig[i] = 1
 
 # The following "fake" class resolves a circular reference issue in the
 # _AMPLEXPORTS datastructure
diff --git a/pyomo/core/expr/expr_pyomo5.py b/pyomo/core/expr/expr_pyomo5.py
--- a/pyomo/core/expr/expr_pyomo5.py
+++ b/pyomo/core/expr/expr_pyomo5.py
@@ -1707,7 +1707,7 @@ def _compute_polynomial_degree(self, result):
         return None
 
     def _apply_operation(self, result):
-        return self._fcn.evaluate( result )     #pragma: no cover
+        return self._fcn.evaluate( result )
 
     def _to_string(self, values, verbose, smap, compute_values):
         return "{0}({1})".format(self.getname(), ", ".join(values))


Test patch:
diff --git a/pyomo/core/tests/unit/test_external.py b/pyomo/core/tests/unit/test_external.py
--- a/pyomo/core/tests/unit/test_external.py
+++ b/pyomo/core/tests/unit/test_external.py
@@ -81,11 +81,36 @@ def test_eval_gsl_function(self):
         if not DLL:
             self.skipTest("Could not find the amplgsl.dll library")
         model = ConcreteModel()
-        model.z_func = ExternalFunction(library=DLL, function="gsl_sf_gamma")
+        model.gamma = ExternalFunction(
+            library=DLL, function="gsl_sf_gamma")
+        model.bessel = ExternalFunction(
+            library=DLL, function="gsl_sf_bessel_Jnu")
         model.x = Var(initialize=3, bounds=(1e-5,None))
-        model.o = Objective(expr=model.z_func(model.x))
+        model.o = Objective(expr=model.gamma(model.x))
         self.assertAlmostEqual(value(model.o), 2.0, 7)
 
+        f = model.bessel.evaluate((0.5, 2.0,))
+        self.assertAlmostEqual(f, 0.5130161365618272, 7)
+
+    def test_eval_fgh_gsl_function(self):
+        DLL = find_GSL()
+        if not DLL:
+            self.skipTest("Could not find the amplgsl.dll library")
+        model = ConcreteModel()
+        model.gamma = ExternalFunction(
+            library=DLL, function="gsl_sf_gamma")
+        model.bessel = ExternalFunction(
+            library=DLL, function="gsl_sf_bessel_Jnu")
+        f,g,h = model.gamma.evaluate_fgh((2.0,))
+        self.assertAlmostEqual(f, 1.0, 7)
+        self.assertAlmostEqual(g, [0.422784335098467], 7)
+        self.assertAlmostEqual(h, [0.8236806608528794], 7)
+
+        f,g,h = model.bessel.evaluate_fgh((2.5, 2.0,), fixed=[1,0])
+        self.assertAlmostEqual(f, 0.223924531469, 7)
+        self.assertAlmostEqual(g, [0.0, 0.21138811435101745], 7)
+        self.assertAlmostEqual(h, [0.0, 0.0, 0.02026349177575621, 0.0], 7)
+
     @unittest.skipIf(not check_available_solvers('ipopt'),
                      "The 'ipopt' solver is not available")
     def test_solve_gsl_function(self):