Repository: Pyomo/pyomo

Issue:
Add `evaluate_fgh` to PythonCallbackFunction API
## Summary

ExternalFunctions through ASL offer calls to `evaluate_fgh` to provide derivative information. We should add this feature to the `PythonCallbackFunction` API as well.

### Rationale

The APIs should match for consistency across different types of ExternalFunctions.




Discussion:

Resolving patch:
diff --git a/pyomo/contrib/trustregion/PyomoInterface.py b/pyomo/contrib/trustregion/PyomoInterface.py
--- a/pyomo/contrib/trustregion/PyomoInterface.py
+++ b/pyomo/contrib/trustregion/PyomoInterface.py
@@ -55,16 +55,16 @@ def exitNode(self, node, values):
         # TODO: support more than PythonCallbackFunctions
         assert isinstance(node._fcn, PythonCallbackFunction)
         #
-        # Note: the first argument to PythonCallbackFunction is the
+        # Note: the last argument to PythonCallbackFunction is the
         # function ID.  Since we are going to complain about constant
-        # parameters, we need to skip the first argument when processing
+        # parameters, we need to skip the last argument when processing
         # the argument list.  This is really not good: we should allow
         # for constant arguments to the functions, and we should relax
         # the restriction that the external functions implement the
         # PythonCallbackFunction API (that restriction leads unfortunate
         # things later; i.e., accessing the private _fcn attribute
         # below).
-        for arg in values[1][1:]:
+        for arg in values[1][:-1]:
             if type(arg) in nonpyomo_leaf_types or arg.is_fixed():
                 # We currently do not allow constants or parameters for
                 # the external functions.
diff --git a/pyomo/core/base/external.py b/pyomo/core/base/external.py
--- a/pyomo/core/base/external.py
+++ b/pyomo/core/base/external.py
@@ -2,7 +2,7 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
 #  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
@@ -12,12 +12,15 @@
 import os
 import types
 import weakref
+from typing import overload
 
 from ctypes import (
     Structure, POINTER, CFUNCTYPE, cdll, byref,
     c_int, c_long, c_ulong, c_double, c_byte, c_char_p, c_void_p )
 
-from pyomo.core.expr.numvalue import native_types, NonNumericValue
+from pyomo.core.expr.numvalue import (
+    native_types, pyomo_constant_types, NonNumericValue, NumericConstant,
+)
 from pyomo.core.expr import current as EXPR
 from pyomo.core.base.component import Component
 from pyomo.core.base.units_container import units
@@ -28,27 +31,103 @@
 
 
 class ExternalFunction(Component):
+    """Interface to an external (non-algebraic) function.
 
-    def __new__(cls, *args, **kwds):
-        if cls != ExternalFunction:
-            return super(ExternalFunction, cls).__new__(cls)
-        if len(args) == 1 and type(args[0]) is types.FunctionType:
-            return PythonCallbackFunction.__new__(PythonCallbackFunction)
-        elif not args and 'library' not in kwds and 'function' in kwds and \
-                type(kwds['function']) is types.FunctionType:
-            return PythonCallbackFunction.__new__(PythonCallbackFunction)
+    :class:`ExternalFunction` provides an interface for declaring
+    general user-provided functions, and then embedding calls to the
+    external functions within Pyomo expressions.
+
+    .. note::
+
+       Just because you can express a Pyomo model with external
+       functions does not mean that the resulting model is solvable.  In
+       particular, linear solvers do not accept external functions.  The
+       AMPL Solver Library (ASL) interface does support external
+       functions for general nonlinear solvers compiled against it, but
+       only allows functions in compiled libraries through the
+       :class:`AMPLExternalFunction` interface.
+
+    """
+    def __new__(cls, *args, **kwargs):
+        if cls is not ExternalFunction:
+            return super().__new__(cls)
+        elif args:
+            return super().__new__(PythonCallbackFunction)
+        elif 'library' not in kwargs and any(
+                kw in kwargs for kw in ('function', 'fgh')):
+            return super().__new__(PythonCallbackFunction)
         else:
-            return AMPLExternalFunction.__new__(AMPLExternalFunction)
+            return super().__new__(AMPLExternalFunction)
+
+    @overload
+    def __init__(self, function=None, gradient=None, hessian=None,
+                 *, fgh=None): ...
+
+    @overload
+    def __init__(self, *, library: str, function: str): ...
+
+    def __init__(self, *args, **kwargs):
+        """Construct a reference to an external function.
+
+        There are two fundamental interfaces supported by
+        :class:`ExternalFunction`: Python callback functions and AMPL
+        external functions.
+
+        **Python callback functions** (:class:`PythonCallbackFunction`
+        interface)
+
+        Python callback functions can be specified one of two ways:
+
+        1. FGH interface:
+
+          A single external function call with a signature matching the
+          :meth:`evaluate_fgh()` method.
+
+        2. Independent functions:
+
+          One to three functions that can evaluate the function value,
+          gradient of the function [partial derivatives] with respect to
+          its inputs, and the Hessian of the function [partial second
+          derivatives].  The ``function`` interface expects a function
+          matching the prototype:
+
+          .. code::
+
+             def function(*args): float
+
+          The ``gradient`` and ``hessian`` interface expect functions
+          matching the prototype:
+
+          .. code::
+
+             def gradient_or_hessian(args, fixed=None): List[float]
 
-    def __init__(self, *args, **kwds):
-        self._units = kwds.pop('units', None)
+          Where ``args`` is a tuple of function arguments and ``fixed``
+          is either None or a list of values equal in length to ``args``
+          indicating which arguments are currently fixed (``True``) or
+          variable (``False``).
+
+        **ASL function libraries** (:class:`AMPLExternalFunction` interface)
+
+        Pyomo can also call functions compiled as part of an AMPL
+        External Function library (see the `User-defined functions
+        <https://www.ampl.com/REFS/HOOKING/#userdefinedfuncs>`_ section
+        in the `Hooking your solver to AMPL
+        <https://www.ampl.com/REFS/hooking3.pdf>`_ report).  Links to
+        these functions are declared by creating an
+        :class:`ExternalFunction` and passing the compiled library name
+        (or path) to the ``library`` keyword and the name of the
+        function to the ``function`` keyword.
+
+        """
+        self._units = kwargs.pop('units', None)
         if self._units is not None:
             self._units = units.get_units(self._units)
-        self._arg_units = kwds.pop('arg_units', None)
+        self._arg_units = kwargs.pop('arg_units', None)
         if self._arg_units is not None:
             self._arg_units = [units.get_units(u) for u in self._arg_units]
-        kwds.setdefault('ctype', ExternalFunction)
-        Component.__init__(self, **kwds)
+        kwargs.setdefault('ctype', ExternalFunction)
+        Component.__init__(self, **kwargs)
         self._constructed = True
         ### HACK ###
         # FIXME: We must declare an _index attribute because
@@ -65,10 +144,31 @@ def get_arg_units(self):
         return self._arg_units
 
     def __call__(self, *args):
+        """Return a Pyomo expression that evaluates this function
+
+        Note that this does not evaluate the external function
+        represented by this :class:`ExternalFunction` component, but
+        rather generates a Pyomo expression object that represents
+        symbolic evaluation of the external function at the specified
+        arguments (which themselves may be Pyomo expression objects).
+
+        Parameters
+        ----------
+        *args:
+            Arguments (constants, variables, expressions) to pass to
+            this External function.
+
+        Returns
+        -------
+        ExternalFunctionExpression or NPV_ExternalFunctionExpression:
+            The new Pyomo expression node.
+
+        """
+
         args_ = []
         for arg in args:
             if type(arg) is types.GeneratorType:
-                args_.extend(val for val in arg)
+                args_.extend(arg)
             else:
                 args_.append(arg)
         #
@@ -81,9 +181,9 @@ def __call__(self, *args):
             try:
                 # Q: Is there a better way to test if a value is an object
                 #    not in native_types and not a standard expression type?
-                if arg.__class__ in native_types or arg.is_expression_type():
-                    pass
-                if not arg.__class__ in native_types and arg.is_potentially_variable():
+                if arg.__class__ in native_types:
+                    continue
+                if arg.is_potentially_variable():
                     pv = True
             except AttributeError:
                 args_[i] = NonNumericValue(arg)
@@ -93,62 +193,168 @@ def __call__(self, *args):
         return EXPR.NPV_ExternalFunctionExpression(args_, self)
 
     def evaluate(self, args):
+
+        """Return the value of the function given the specified arguments
+
+        Parameters
+        ----------
+        args: Iterable
+            Iterable containing the arguments to pass to the external
+            function.  Non-native type elements will be converted to a
+            native value using the :py:func:`value()` function.
+
+        Returns
+        -------
+        float
+            The return value of the function evaluated at `args`
+        """
+        args_ = [arg if arg.__class__ in native_types else value(arg)
+                 for arg in args]
+        return self._evaluate(args_, None, 0)[0]
+
+    def evaluate_fgh(self, args, fixed=None, fgh=2):
+        """Evaluate the function and gradients given the specified arguments
+
+        This evaluates the function given the specified arguments
+        returning a 3-tuple of (function value [f], list of first partial
+        derivatives [g], and the upper triangle of the Hessian matrix
+        [h]).
+
+        Parameters
+        ----------
+        args: Iterable
+            Iterable containing the arguments to pass to the external
+            function.  Non-native type elements will be converted to a
+            native value using the :py:func:`value()` function.
+
+        fixed: Optional[List[bool]]
+            List of values indicating if the corresponding argument
+            value is fixed.  Any fixed indices are guaranteed to return
+            0 for first and second derivatives, regardless of what is
+            computed by the external function.
+
+        fgh: {0, 1, 2}
+            What evaluations to return:
+
+            * **0**: just return function evaluation
+            * **1**: return function and first derivatives
+            * **2**: return function, first derivatives, and hessian matrix
+
+            Any return values not requested will be `None`.
+
+        Returns
+        -------
+        f: float
+            The return value of the function evaluated at `args`
+        g: List[float] or None
+            The list of first partial derivatives
+        h: List[float] or None
+            The upper-triangle of the Hessian matrix (second partial
+            derivatives), stored column-wise.  Element :math:`H_{i,j}`
+            (with :math:`0 <= i <= j < N` are mapped using
+            :math:`h[i + j*(j + 1)/2] == H_{i,j}`.
+
+        """
+        args_ = [arg if arg.__class__ in native_types else value(arg)
+                 for arg in args]
+        # Note: this is passed-by-reference, and the args_ list may be
+        # changed by _evaluate (e.g., for PythonCallbackFunction).
+        # Remember the original length of the list.
+        N = len(args_)
+        f, g, h = self._evaluate(args_, fixed, fgh)
+        # Guarantee the return value behavior documented in the docstring
+        if fgh == 2:
+            n = N - 1
+            if len(h) - 1 != n + n*(n+1)//2:
+                raise RuntimeError(
+                    f"External function '{self.name}' returned an invalid "
+                    f"Hessian matrix (expected {n + n*(n+1)//2 + 1}, "
+                    f"received {len(h)})")
+        else:
+            h = None
+        if fgh >= 1:
+            if len(g) != N:
+                raise RuntimeError(
+                    f"External function '{self.name}' returned an invalid "
+                    f"derivative vector (expected {N}, "
+                    f"received {len(g)})")
+        else:
+            g = None
+        # Note: the ASL does not require clients to honor the fixed flag
+        # (allowing them to return non-0 values for the derivative with
+        # respect to a fixed numeric value).  We will allow clients to
+        # be similarly lazy and enforce the fixed flag here.
+        if fixed is not None:
+            if fgh >= 1:
+                for i, v in enumerate(fixed):
+                    if not v:
+                        continue
+                    g[i] = 0
+            if fgh >= 2:
+                for i, v in enumerate(fixed):
+                    if not v:
+                        continue
+                    for j in range(N):
+                        if i <= j:
+                            h[i + (j*(j + 1))//2] = 0
+                        else:
+                            h[j + (i*(i + 1))//2] = 0
+        return f, g, h
+
+    def _evaluate(self, args, fixed, fgh):
+        """Derived class implementation to evaluate an external function.
+
+        The API follows :meth:`evaluate_fgh()`, with the exception that
+        the `args` iterable is guaranteed to already have been reduced
+        to concrete values (it will not have Pyomo components or
+        expressions).
+        """
         raise NotImplementedError(
-            "General external functions can not be evaluated within Python." )
+            f"{type(self)} did not implement _evaluate()" )
 
 
 class AMPLExternalFunction(ExternalFunction):
 
-    def __init__(self, *args, **kwds):
+    def __init__(self, *args, **kwargs):
         if args:
             raise ValueError(
                 "AMPLExternalFunction constructor does not support "
                 "positional arguments" )
-        self._library = kwds.pop('library', None)
-        self._function = kwds.pop('function', None)
+        self._library = kwargs.pop('library', None)
+        self._function = kwargs.pop('function', None)
         self._known_functions = None
         self._so = None
-        ExternalFunction.__init__(self, *args, **kwds)
+        ExternalFunction.__init__(self, *args, **kwargs)
 
     def __getstate__(self):
-        state = super(AMPLExternalFunction, self).__getstate__()
+        state = super().__getstate__()
         # Remove reference to loaded library (they are not copyable or
         # picklable)
         state['_so'] = state['_known_functions'] = None
         return state
 
-    def _evaluate(self, args, fgh, fixed):
+    def _evaluate(self, args, fixed, fgh):
         if self._so is None:
             self.load_library()
         if self._function not in self._known_functions:
             raise RuntimeError(
-                "Error: external function %s was not registered within "
+                "Error: external function '%s' was not registered within "
                 "external library %s.\n\tAvailable functions: (%s)"
                 % ( self._function, self._library,
                     ', '.join(self._known_functions.keys()) ) )
         #
-        args_ = []
-        for arg in args:
-            if arg.__class__ is NonNumericValue:
-                args_.append(arg.value)
-            else:
-                args_.append(arg)
-        arglist = _ARGLIST( args_, fgh, fixed )
+        N = len(args)
+        arglist = _ARGLIST(args, fgh, fixed)
         fcn = self._known_functions[self._function][0]
         f = fcn(byref(arglist))
-        return f, arglist
-
-    def evaluate(self, args):
-        return self._evaluate(args, 0, None)[0]
-
-    def evaluate_fgh(self, args, fixed=None):
-        if fixed is None:
-            fixed = tuple( arg.__class__ in native_types or arg.is_fixed()
-                           for arg in args )
-        N = len(args)
-        f, arglist = self._evaluate(args, 2, fixed)
-        g = [arglist.derivs[i] for i in range(N)]
-        h = [arglist.hes[i] for i in range((N+N**2)//2)]
+        if fgh >= 1:
+            g = [arglist.derivs[i] for i in range(N)]
+        else:
+            g = None
+        if fgh >= 2:
+            h = [arglist.hes[i] for i in range((N + N**2)//2)]
+        else:
+            h = None
         return f, g, h
 
     def load_library(self):
@@ -201,59 +407,150 @@ def _pprint(self):
         )
 
 
+class _PythonCallbackFunctionID(NumericConstant):
+    """A specialized NumericConstant to preserve FunctionIDs through deepcopy.
+
+    Function IDs are effectively pointers back to the
+    PythonCallbackFunction.  As such, we need special handling to
+    maintain / preserve the correct linkages through deepcopy (and
+    model.clone()).
+
+    """
+    __slots__ = ()
+
+    def is_constant(self):
+        # Return False so this object is not simplified out of expressions
+        return False
+
+    def __getstate__(self):
+        state = super().__getstate__()
+        state['value'] = PythonCallbackFunction.global_registry[
+            state['value']]()
+        return state
+
+    def __setstate__(self, state):
+        state['value'] = state['value']._fcn_id
+        super().__setstate__(state)
+
+pyomo_constant_types.add(_PythonCallbackFunctionID)
+
+
 class PythonCallbackFunction(ExternalFunction):
-    global_registry = {}
+    global_registry = []
+    global_id_to_fid = {}
 
     @classmethod
     def register_instance(cls, instance):
-        _id = len(cls.global_registry)
-        cls.global_registry[_id] = weakref.ref(instance)
-        return _id
-
-    def __init__(self, *args, **kwds):
-        if args:
-            self._fcn = args[0]
-            if len(args) > 1:
+        # Ideally, we would just use the id() as the function id, but
+        # Python id() is more than 32-bits, which can cause problems for
+        # the ASL.  We will map id() to a "small" integer and use that
+        # small integer as the "function id".
+        _id = id(instance)
+        if _id in cls.global_id_to_fid:
+            prev_fid = cls.global_id_to_fid[_id]
+            prev_obj = cls.global_registry[prev_fid]
+            if prev_obj() is not None:
+                assert prev_obj() is instance
+                return prev_fid
+        cls.global_id_to_fid[_id] = _fid = len(cls.global_registry)
+        cls.global_registry.append(weakref.ref(instance))
+        return _fid
+
+    def __init__(self, *args, **kwargs):
+        for i, kw in enumerate(('function', 'gradient', 'hessian')):
+            if len(args) <= i:
+                break
+            if kw in kwargs:
                 raise ValueError(
-                    "PythonCallbackFunction constructor only supports a "
-                    "single positional positional arguments" )
-        if not args:
-            self._fcn = kwds.pop('function')
+                    "Duplicate definition of external function through "
+                    f"positional and keyword ('{kw}=') arguments")
+            kwargs[kw] = args[i]
+        if len(args) > 3:
+            raise ValueError(
+                "PythonCallbackFunction constructor only supports "
+                "0 - 3 positional arguments" )
+        self._fcn = kwargs.pop('function', None)
+        self._grad = kwargs.pop('gradient', None)
+        self._hess = kwargs.pop('hessian', None)
+        self._fgh = kwargs.pop('fgh', None)
+        if self._fgh is not None and any((self._fcn, self._grad, self._hess)):
+            raise ValueError(
+                "Cannot specify 'fgh' with any of "
+                "{'function', 'gradient', hessian'}")
 
         # There is an implicit first argument (the function pointer), we
         # need to add that to the arg_units
-        arg_units = kwds.get('arg_units', None)
+        arg_units = kwargs.get('arg_units', None)
         if arg_units is not None:
-            kwds['arg_units'] = [None] + list(arg_units)
+            kwargs['arg_units'] = list(arg_units)
+            kwargs['arg_units'].append(None)
 
+        # TODO: complete/distribute the pyomo_socket_server /
+        # pyomo_ampl.so AMPL extension
         self._library = 'pyomo_ampl.so'
         self._function = 'pyomo_socket_server'
-        ExternalFunction.__init__(self, *args, **kwds)
+        ExternalFunction.__init__(self, *args, **kwargs)
         self._fcn_id = PythonCallbackFunction.register_instance(self)
 
-    def __call__(self, *args):
-        return super(PythonCallbackFunction, self).__call__(self._fcn_id, *args)
+    def __getstate__(self):
+        state = super().__getstate__()
+        state['_fcn_id'] = self
+        return state
 
-    def evaluate(self, args):
-        if args.__class__ is types.GeneratorType:
-            args = tuple(args)
-        args_ = []
-        for arg in args:
-            if arg.__class__ is NonNumericValue:
-                args_.append(arg.value)
-            else:
-                args_.append(arg)
-        _id = args_[0]
+    def __setstate__(self, state):
+        state['_fcn_id'] = PythonCallbackFunction.register_instance(
+            state['_fcn_id'])
+        super().__setstate__(state)
+
+    def __call__(self, *args):
+        # NOTE: we append the Function ID to the END of the argument
+        # list because it is easier to update the gradient / hessian
+        # results
+        return super().__call__(*args, _PythonCallbackFunctionID(self._fcn_id))
+
+    def _evaluate(self, args, fixed, fgh):
+        # Remove the fcn_id
+        _id = args.pop()
+        if fixed is not None:
+            fixed = fixed[:-1]
         if _id != self._fcn_id:
             raise RuntimeError(
                 "PythonCallbackFunction called with invalid Global ID" )
-        return self._fcn(*args_[1:])
+        if self._fgh is not None:
+            f, g, h = self._fgh(args, fgh, fixed)
+        else:
+            f = self._fcn(*args)
+            if fgh >= 1:
+                if self._grad is None:
+                    raise RuntimeError(
+                        f"ExternalFunction '{self.name}' was not defined "
+                        "with a gradient callback.  Cannot evaluate the "
+                        "derivative of the function")
+                g = self._grad(args, fixed)
+            else:
+                g = None
+            if fgh == 2:
+                if self._hess is None:
+                    raise RuntimeError(
+                        f"ExternalFunction '{self.name}' was not defined "
+                        "with a Hessian callback.  Cannot evaluate the "
+                        "second derivative of the function")
+                h = self._hess(args, fixed)
+            else:
+                h = None
+        # Update g,h to reflect the function id (which by definition is
+        # always fixed)
+        if g is not None:
+            g.append(0)
+        if h is not None:
+            h.extend([0]*(len(args)+1))
+        return f, g, h
 
     def _pprint(self):
         return (
             [ ('function', self._fcn.__qualname__),
               ('units', str(self._units)),
-              ('arg_units', [ str(u) for u in self._arg_units[1:] ]
+              ('arg_units', [ str(u) for u in self._arg_units[:-1] ]
                if self._arg_units is not None else None),
             ],
             (), None, None
@@ -306,15 +603,15 @@ class _ARGLIST(Structure):
         ]
 
     def __init__(self, args, fgh=0, fixed=None):
-        super(_ARGLIST, self).__init__()
+        super().__init__()
         N = len(args)
         self.n = self.nr = N
         self.at = (c_int*N)()
         self.ra = (c_double*N)()
         self.sa = None
-        if fgh > 0:
+        if fgh >= 1:
             self.derivs = (c_double*N)(0.)
-        if fgh > 1:
+        if fgh >= 2:
             self.hes = (c_double*((N+N*N)//2))(0.)
 
         for i,v in enumerate(args):
@@ -322,6 +619,7 @@ def __init__(self, args, fgh=0, fixed=None):
             self.ra[i] = v
 
         if fixed:
+            # This has to be revisited if nr != ra
             self.dig = (c_byte*N)(0)
             for i,v in enumerate(fixed):
                 if v:
diff --git a/pyomo/core/expr/calculus/diff_with_pyomo.py b/pyomo/core/expr/calculus/diff_with_pyomo.py
--- a/pyomo/core/expr/calculus/diff_with_pyomo.py
+++ b/pyomo/core/expr/calculus/diff_with_pyomo.py
@@ -312,13 +312,13 @@ def _diff_ExternalFunctionExpression(node, val_dict, der_dict):
 
     Parameters
     ----------
-    node: pyomo.core.expr.numeric_expr.ProductExpression
+    node: pyomo.core.expr.numeric_expr.ExternalFunctionExpression
     val_dict: ComponentMap
     der_dict: ComponentMap
     """
     der = der_dict[node]
     vals = tuple(val_dict[i] for i in node.args)
-    derivs = node._fcn.evaluate_fgh(vals)[1]
+    derivs = node._fcn.evaluate_fgh(vals, fgh=1)[1]
     for ndx, arg in enumerate(node.args):
         der_dict[arg] += der * derivs[ndx]
 


Test patch:
diff --git a/pyomo/core/tests/unit/test_external.py b/pyomo/core/tests/unit/test_external.py
--- a/pyomo/core/tests/unit/test_external.py
+++ b/pyomo/core/tests/unit/test_external.py
@@ -8,29 +8,92 @@
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 #
-
+import os
+import shutil
+import sys
 from io import StringIO
 
 import pyomo.common.unittest as unittest
 
 from pyomo.common.getGSL import find_GSL
-from pyomo.environ import ConcreteModel, Var, Objective, SolverFactory, value
-from pyomo.core.base.external import (PythonCallbackFunction,
-                                      ExternalFunction,
-                                      AMPLExternalFunction)
+from pyomo.common.tempfiles import TempfileManager
+from pyomo.environ import (
+    ConcreteModel, Block, Var, Objective, Expression, SolverFactory, value,
+    Param,
+)
+from pyomo.core.base.external import (
+    PythonCallbackFunction, ExternalFunction, AMPLExternalFunction,
+)
 from pyomo.core.base.units_container import pint_available, units
+from pyomo.core.expr.numeric_expr import (
+    ExternalFunctionExpression, NPV_ExternalFunctionExpression,
+)
 from pyomo.opt import check_available_solvers
 
-def _g(*args):
+def _count(*args):
     return len(args)
 
-def _h(*args):
+def _sum(*args):
     return 2 + sum(args)
 
+def _f(x, y, z):
+    return x**2 + 3*x*y + x*y*z**2
+
+def _g(args, fixed):
+    x, y, z = args[:3]
+    return [
+        2*x + 3*y + y*z**2,
+        3*x + x*z**2,
+        2*x*y*z,
+    ]
+
+def _h(args, fixed):
+    x, y, z = args[:3]
+    return [
+        2,
+        3+z**2,     0,
+        2*y*z,  2*x*z, 2*x*y,
+    ]
+
+def _g_bad(args, fixed):
+    x, y, z = args[:3]
+    return [
+        2*x + 3*y + y*z**2,
+        3*x + x*z**2,
+        2*x*y*z,
+        0,
+    ]
+
+def _h_bad(args, fixed):
+    x, y, z = args[:3]
+    return [
+        #2,
+        3+z**2,     0,
+        2*y*z,  2*x*z, 2*x*y,
+    ]
+
+def _fgh(args, fixed, fgh):
+    return _f(*args), _g(args, fixed), _h(args, fixed)
+
 class TestPythonCallbackFunction(unittest.TestCase):
+    def test_constructor_errors(self):
+        m = ConcreteModel()
+        with self.assertRaisesRegex(
+                ValueError, "Duplicate definition of external function "
+                r"through positional and keyword \('function='\)"):
+            m.f = ExternalFunction(_count, function=_count)
+        with self.assertRaisesRegex(
+                ValueError, "PythonCallbackFunction constructor only "
+                "supports 0 - 3 positional arguments"):
+            m.f = ExternalFunction(1, 2, 3, 4)
+        with self.assertRaisesRegex(
+                ValueError, "Cannot specify 'fgh' with any of "
+                "{'function', 'gradient', hessian'}"):
+            m.f = ExternalFunction(_count, fgh=_fgh)
+
     def test_call_countArgs(self):
         m = ConcreteModel()
-        m.f = ExternalFunction(_g)
+        m.f = ExternalFunction(_count)
         self.assertIsInstance(m.f, PythonCallbackFunction)
         self.assertEqual(value(m.f()), 0)
         self.assertEqual(value(m.f(2)), 1)
@@ -38,15 +101,150 @@ def test_call_countArgs(self):
 
     def test_call_sumfcn(self):
         m = ConcreteModel()
-        m.f = ExternalFunction(_h)
+        m.f = ExternalFunction(_sum)
         self.assertIsInstance(m.f, PythonCallbackFunction)
         self.assertEqual(value(m.f()), 2.0)
         self.assertEqual(value(m.f(1)), 3.0)
         self.assertEqual(value(m.f(1,2)), 5.0)
 
+    def test_evaluate_fgh_fgh(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(fgh=_fgh)
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id))
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             3*5 + 5*11**2,
+                             2*5*7*11,
+                             0])
+        self.assertEqual(h, [
+                  2,
+            3+11**2,      0,
+             2*7*11, 2*5*11, 2*5*7,
+                  0,      0,     0,  0,
+        ])
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id),
+                                   fixed=[0, 1, 0, 1])
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             0,
+                             2*5*7*11,
+                             0])
+        self.assertEqual(h, [
+                  2,
+                  0,      0,
+             2*7*11,      0, 2*5*7,
+                  0,      0,     0,  0,
+        ])
+
+    def test_evaluate_fgh_f_g_h(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_f, _g, _h)
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id))
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             3*5 + 5*11**2,
+                             2*5*7*11,
+                             0])
+        self.assertEqual(h, [
+                  2,
+            3+11**2,      0,
+             2*7*11, 2*5*11, 2*5*7,
+                  0,      0,     0,  0,
+        ])
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id),
+                                   fixed=[0, 1, 0, 1])
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             0,
+                             2*5*7*11,
+                             0])
+        self.assertEqual(h, [
+                  2,
+                  0,      0,
+             2*7*11,      0, 2*5*7,
+                  0,      0,     0,  0,
+        ])
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=1)
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             3*5 + 5*11**2,
+                             2*5*7*11,
+                             0])
+        self.assertIsNone(h)
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=0)
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertIsNone(g)
+        self.assertIsNone(h)
+
+    def test_evaluate_fgh_f_g(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_f, _g)
+        with self.assertRaisesRegex(
+                RuntimeError, "ExternalFunction 'f' was not defined "
+                "with a Hessian callback."):
+            f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id))
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=1)
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertEqual(g, [2*5 + 3*7 + 7*11**2,
+                             3*5 + 5*11**2,
+                             2*5*7*11,
+                             0])
+        self.assertIsNone(h)
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=0)
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertIsNone(g)
+        self.assertIsNone(h)
+
+    def test_evaluate_fgh_f(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_f)
+        with self.assertRaisesRegex(
+                RuntimeError, "ExternalFunction 'f' was not defined "
+                "with a gradient callback."):
+            f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id))
+
+        with self.assertRaisesRegex(
+                RuntimeError, "ExternalFunction 'f' was not defined "
+                "with a gradient callback."):
+            f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=1)
+
+        f, g, h = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=0)
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+        self.assertIsNone(g)
+        self.assertIsNone(h)
+
+    def test_evaluate_errors(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_f, _g_bad, _h_bad)
+        f = m.f.evaluate((5, 7, 11, m.f._fcn_id))
+        self.assertEqual(f, 5**2 + 3*5*7 + 5*7*11**2)
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "PythonCallbackFunction called with invalid Global ID"):
+            f = m.f.evaluate((5, 7, 11, -1))
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "External function 'f' returned an invalid "
+                r"derivative vector \(expected 4, received 5\)"):
+            f = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=1)
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "External function 'f' returned an invalid "
+                r"Hessian matrix \(expected 10, received 9\)"):
+            f = m.f.evaluate_fgh((5, 7, 11, m.f._fcn_id), fgh=2)
+
     def test_getname(self):
         m = ConcreteModel()
-        m.f = ExternalFunction(_h)
+        m.f = ExternalFunction(_sum)
         self.assertIsInstance(m.f, PythonCallbackFunction)
         self.assertEqual(m.f.name, "f")
         self.assertEqual(m.f.local_name, "f")
@@ -63,31 +261,109 @@ def test_getname(self):
     def test_extra_kwargs(self):
         m = ConcreteModel()
         with self.assertRaises(ValueError):
-            m.f = ExternalFunction(_g, this_should_raise_error='foo')
+            m.f = ExternalFunction(_count, this_should_raise_error='foo')
+
+    def test_clone(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_sum)
+        m.x = Var(initialize=3)
+        m.y = Var(initialize=5)
+        m.e = Expression(expr=m.f(m.x, m.y))
+        self.assertIsInstance(m.f, PythonCallbackFunction)
+        self.assertEqual(m.f._fcn_id, m.e.arg(0).arg(-1).value)
+        self.assertEqual(value(m.e), 10)
+
+        i = m.clone()
+        self.assertIsInstance(i.f, PythonCallbackFunction)
+        self.assertIsNot(i.f, m.f)
+        self.assertIsNot(i.e, m.e)
+        self.assertIsNot(i.e.arg(0), m.e.arg(0))
+        self.assertEqual(i.f._fcn_id, i.e.arg(0).arg(-1).value)
+        self.assertNotEqual(i.f._fcn_id, m.f._fcn_id)
+        self.assertEqual(value(i.e), 10)
+
+    def test_partial_clone(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_sum)
+        m.x = Var(initialize=3)
+        m.y = Var(initialize=5)
+        m.b = Block()
+        m.b.e = Expression(expr=m.f(m.x, m.y))
+        self.assertIsInstance(m.f, PythonCallbackFunction)
+        self.assertEqual(m.f._fcn_id, m.b.e.arg(0).arg(-1).value)
+        self.assertEqual(value(m.b.e), 10)
+
+        m.c = m.b.clone()
+        self.assertIsNot(m.b.e, m.c.e)
+        self.assertIsNot(m.b.e.arg(0), m.c.e.arg(0))
+        self.assertEqual(m.f._fcn_id, m.b.e.arg(0).arg(-1).value)
+        self.assertEqual(m.f._fcn_id, m.c.e.arg(0).arg(-1).value)
+        self.assertEqual(value(m.c.e), 10)
+
+        # save / restore should not change the fcn_id
+        _fcn_id = m.f._fcn_id
+        m.f.__setstate__(m.f.__getstate__())
+        self.assertEqual(m.f._fcn_id, _fcn_id)
+
+        self.assertIsNot(m.b.e, m.c.e)
+        self.assertIsNot(m.b.e.arg(0), m.c.e.arg(0))
+        self.assertEqual(m.f._fcn_id, m.b.e.arg(0).arg(-1).value)
+        self.assertEqual(m.f._fcn_id, m.c.e.arg(0).arg(-1).value)
+        self.assertEqual(value(m.c.e), 10)
+
+    def test_properties(self):
+        m = ConcreteModel()
+        m.f = ExternalFunction(_sum)
+        e = m.f()
+        self.assertIsInstance(e, NPV_ExternalFunctionExpression)
+        self.assertFalse(e.is_constant())
+        self.assertTrue(e.is_fixed())
+        self.assertFalse(e.is_potentially_variable())
+
+        # This is just a check for coverage: the expression is_constant
+        # catches that the function ID is a NumericConstant and doesn't
+        # call the method.
+        self.assertFalse(e.arg(0).is_constant())
+        self.assertTrue(e.arg(0).is_fixed())
+        self.assertFalse(e.arg(0).is_potentially_variable())
+
+        m.p = Param(initialize=1)
+        e = m.f(m.p)
+        self.assertIsInstance(e, NPV_ExternalFunctionExpression)
+        self.assertFalse(e.is_constant())
+        self.assertTrue(e.is_fixed())
+        self.assertFalse(e.is_potentially_variable())
+
+        m.x = Var(initialize=1)
+        e = m.f(m.p, m.x)
+        self.assertIsInstance(e, ExternalFunctionExpression)
+        self.assertFalse(e.is_constant())
+        self.assertFalse(e.is_fixed())
+        self.assertTrue(e.is_potentially_variable())
 
     def test_pprint(self):
         m = ConcreteModel()
-        m.h = ExternalFunction(_g)
+        m.h = ExternalFunction(_count)
 
         out = StringIO()
         m.pprint(ostream=out)
         self.assertEqual(out.getvalue().strip(), """
 1 ExternalFunction Declarations
-    h : function=_g, units=None, arg_units=None
+    h : function=_count, units=None, arg_units=None
 
 1 Declarations: h
         """.strip())
 
         if not pint_available:
             return
-        m.i = ExternalFunction(function=_h,
+        m.i = ExternalFunction(function=_sum,
                                units=units.kg, arg_units=[units.m, units.s])
         out = StringIO()
         m.pprint(ostream=out)
         self.assertEqual(out.getvalue().strip(), """
 2 ExternalFunction Declarations
-    h : function=_g, units=None, arg_units=None
-    i : function=_h, units=kg, arg_units=['m', 's']
+    h : function=_count, units=None, arg_units=None
+    i : function=_sum, units=kg, arg_units=['m', 's']
 
 2 Declarations: h i
         """.strip())
@@ -117,6 +393,36 @@ def test_getname(self):
         self.assertEqual(M.m.f.getname(), "f")
         self.assertEqual(M.m.f.getname(True), "m.f")
 
+    @unittest.skipIf(sys.platform.lower().startswith('win'),
+                     "Cannot (easily) unload a DLL in Windows, so "
+                     "cannot clean up the 'temporary' DLL")
+    def test_load_local_asl_library(self):
+        DLL = find_GSL()
+        if not DLL:
+            self.skipTest("Could not find the amplgsl.dll library")
+
+        LIB = 'test_pyomo_external_gsl.dll'
+
+        model = ConcreteModel()
+        model.gamma = ExternalFunction(
+            library=LIB, function="gsl_sf_gamma")
+        model.x = Var(initialize=3, bounds=(1e-5,None))
+        model.o = Objective(expr=model.gamma(model.x))
+
+        with TempfileManager.new_context() as tempfile:
+            dname = tempfile.mkdtemp()
+            shutil.copyfile(DLL, os.path.join(dname, LIB))
+            # Without changing directories, the load should fail
+            with self.assertRaises(OSError):
+                value(model.o)
+            # Changing directories should pick up the library
+            try:
+                orig_dir = os.getcwd()
+                os.chdir(dname)
+                self.assertAlmostEqual(value(model.o), 2.0, 7)
+            finally:
+                os.chdir(orig_dir)
+
     def test_eval_gsl_function(self):
         DLL = find_GSL()
         if not DLL:
@@ -133,6 +439,18 @@ def test_eval_gsl_function(self):
         f = model.bessel.evaluate((0.5, 2.0,))
         self.assertAlmostEqual(f, 0.5130161365618272, 7)
 
+    def test_eval_gsl_error(self):
+        DLL = find_GSL()
+        if not DLL:
+            self.skipTest("Could not find the amplgsl.dll library")
+        model = ConcreteModel()
+        model.bogus = ExternalFunction(
+            library=DLL, function="bogus_function")
+        with self.assertRaisesRegex(
+                RuntimeError, "Error: external function 'bogus_function' was "
+                "not registered within external library(?s:.*)gsl_sf_gamma"):
+            f = model.bogus.evaluate((1,))
+
     def test_eval_fgh_gsl_function(self):
         DLL = find_GSL()
         if not DLL:
@@ -140,18 +458,67 @@ def test_eval_fgh_gsl_function(self):
         model = ConcreteModel()
         model.gamma = ExternalFunction(
             library=DLL, function="gsl_sf_gamma")
+        model.beta = ExternalFunction(
+            library=DLL, function="gsl_sf_beta")
         model.bessel = ExternalFunction(
             library=DLL, function="gsl_sf_bessel_Jnu")
+
         f,g,h = model.gamma.evaluate_fgh((2.0,))
         self.assertAlmostEqual(f, 1.0, 7)
         self.assertListsAlmostEqual(g, [0.422784335098467], 7)
         self.assertListsAlmostEqual(h, [0.8236806608528794], 7)
 
+        f,g,h = model.beta.evaluate_fgh((2.5, 2.0,), fixed=[1,1])
+        self.assertAlmostEqual(f, 0.11428571428571432, 7)
+        self.assertListsAlmostEqual(g, [0.0, 0.0], 7)
+        self.assertListsAlmostEqual(h, [0.0, 0.0, 0.0], 7)
+
+        f,g,h = model.beta.evaluate_fgh((2.5, 2.0,), fixed=[0,1])
+        self.assertAlmostEqual(f, 0.11428571428571432, 7)
+        self.assertListsAlmostEqual(
+            g, [-0.07836734693877555, 0.0], 7)
+        self.assertListsAlmostEqual(
+            h, [0.08135276967930034, 0.0, 0.0], 7)
+
+        f,g,h = model.beta.evaluate_fgh((2.5, 2.0,))
+        self.assertAlmostEqual(f, 0.11428571428571432, 7)
+        self.assertListsAlmostEqual(
+            g, [-0.07836734693877555, -0.11040989614412142], 7)
+        self.assertListsAlmostEqual(
+            h, [0.08135276967930034, 0.0472839170086535, 0.15194654464270113],
+            7)
+
+        f,g,h = model.beta.evaluate_fgh((2.5, 2.0,), fgh=1)
+        self.assertAlmostEqual(f, 0.11428571428571432, 7)
+        self.assertListsAlmostEqual(
+            g, [-0.07836734693877555, -0.11040989614412142], 7)
+        self.assertIsNone(h)
+
+        f,g,h = model.beta.evaluate_fgh((2.5, 2.0,), fgh=0)
+        self.assertAlmostEqual(f, 0.11428571428571432, 7)
+        self.assertIsNone(g)
+        self.assertIsNone(h)
+
         f,g,h = model.bessel.evaluate_fgh((2.5, 2.0,), fixed=[1,0])
         self.assertAlmostEqual(f, 0.223924531469, 7)
         self.assertListsAlmostEqual(g, [0.0, 0.21138811435101745], 7)
         self.assertListsAlmostEqual(h, [0.0, 0.0, 0.02026349177575621], 7)
 
+        # Note: Not all AMPL-GSL functions honor the fixed flag
+        # (notably, gamma and bessel do not as of 12/2021).  We will
+        # test that our interface corrects that
+
+        f,g,h = model.gamma.evaluate_fgh((2.0,), fixed=[1])
+        self.assertAlmostEqual(f, 1.0, 7)
+        self.assertListsAlmostEqual(g, [0.0], 7)
+        self.assertListsAlmostEqual(h, [0.0], 7)
+
+        f,g,h = model.bessel.evaluate_fgh((2.5, 2.0,), fixed=[1,1])
+        self.assertAlmostEqual(f, 0.223924531469, 7)
+        self.assertListsAlmostEqual(g, [0.0, 0.0], 7)
+        self.assertListsAlmostEqual(h, [0.0, 0.0, 0.0], 7)
+
+
     @unittest.skipIf(not check_available_solvers('ipopt'),
                      "The 'ipopt' solver is not available")
     def test_solve_gsl_function(self):