Repository: Pyomo/pyomo

Issue:
Pyomo should be able to load external functions from .so file in LD_LIBRARY_PATH 
Currently pyomo only loads external function property definitions from a phys_prop.so file in the current directory. It should search LD_LIBRARY_PATH (or equivalent) location for this file.

Pyomo should be able to load external functions from .so file in LD_LIBRARY_PATH 
Currently pyomo only loads external function property definitions from a phys_prop.so file in the current directory. It should search LD_LIBRARY_PATH (or equivalent) location for this file.



Discussion:
@jsiirola you've done a lot of work on external functions since this issue was opened. Has this been resolved?
No.  To my knowledge, no work has been done on this.

The trick is that in the usual mode of operation, Pyomo is not responsible for loading the library.  The library is loaded by the ASL (i.e., is compiled into the solver), and we don't have control over how the ASL located the library.  That said, I think we *can* mock this up by attempting to convert relative library path to an absolute path (by searching LD_LIBRARY_PATH) before generating the NL file.  The open design question is *when* to do that expansion: when the external function is declared or when the NL file is generated.  I think the former will produce more consistent (and understandable to a user) behavior.  However, it could cause problems in parallel environments where the model is generated on one node, but solved somewhere else (the LD_LIBRARY_PATH would be checked locally and not remotely).  
@jsiirola @blnicho - It's been a few years. Any idea if work has been done to resolve this?
@jsiirola you've done a lot of work on external functions since this issue was opened. Has this been resolved?
No.  To my knowledge, no work has been done on this.

The trick is that in the usual mode of operation, Pyomo is not responsible for loading the library.  The library is loaded by the ASL (i.e., is compiled into the solver), and we don't have control over how the ASL located the library.  That said, I think we *can* mock this up by attempting to convert relative library path to an absolute path (by searching LD_LIBRARY_PATH) before generating the NL file.  The open design question is *when* to do that expansion: when the external function is declared or when the NL file is generated.  I think the former will produce more consistent (and understandable to a user) behavior.  However, it could cause problems in parallel environments where the model is generated on one node, but solved somewhere else (the LD_LIBRARY_PATH would be checked locally and not remotely).  
@jsiirola @blnicho - It's been a few years. Any idea if work has been done to resolve this?

Resolving patch:
diff --git a/pyomo/core/base/external.py b/pyomo/core/base/external.py
--- a/pyomo/core/base/external.py
+++ b/pyomo/core/base/external.py
@@ -18,6 +18,7 @@
     Structure, POINTER, CFUNCTYPE, cdll, byref,
     c_int, c_long, c_ulong, c_double, c_byte, c_char_p, c_void_p )
 
+from pyomo.common.fileutils import find_library
 from pyomo.core.expr.numvalue import (
     native_types, native_numeric_types, pyomo_constant_types,
     NonNumericValue, NumericConstant,
@@ -325,6 +326,16 @@ def __init__(self, *args, **kwargs):
         self._function = kwargs.pop('function', None)
         self._known_functions = None
         self._so = None
+        # Convert the specified library name to an absolute path, and
+        # warn the user if we couldn't find it
+        if self._library is not None:
+            _lib = find_library(self._library)
+            if _lib is not None:
+                self._library = _lib
+            else:
+                logger.warning(
+                    'Defining AMPL external function, but cannot locate '
+                    f'specified library "{self._library}"')
         ExternalFunction.__init__(self, *args, **kwargs)
 
     def __getstate__(self):
@@ -359,19 +370,18 @@ def _evaluate(self, args, fixed, fgh):
         return f, g, h
 
     def load_library(self):
-        try:
-            self._so = cdll.LoadLibrary(self._library)
-        except OSError:
-            # On Linux, it is uncommon for "." to be in the
-            # LD_LIBRARY_PATH, so if things fail to load, attempt to
-            # locate the library via a relative path
-            try:
-                self._so = cdll.LoadLibrary(os.path.join('.',self._library))
-            except OSError:
-                # Re-try with the original library name and allow the
-                # exception to propagate up.
-                self._so = cdll.LoadLibrary(self._library)
-
+        # Note that the library was located in __init__ and converted to
+        # an absolute path (including checking the CWD).  However, we
+        # previously tested that changing the environment (i.e.,
+        # changing directories) between defining the ExternalFunction
+        # and loading it would cause the library to still be correctly
+        # loaded.  We will re-search for the library here.  If it was
+        # previously found, we will be searching for an absolute path
+        # (and the same path will be found again).
+        _abs_lib = find_library(self._library)
+        if _abs_lib is not None:
+            self._library = _abs_lib
+        self._so = cdll.LoadLibrary(self._library)
         self._known_functions = {}
         AE = _AMPLEXPORTS()
         AE.ASLdate = 20160307
diff --git a/pyomo/repn/plugins/ampl/ampl_.py b/pyomo/repn/plugins/ampl/ampl_.py
--- a/pyomo/repn/plugins/ampl/ampl_.py
+++ b/pyomo/repn/plugins/ampl/ampl_.py
@@ -21,6 +21,7 @@
 import time
 from math import isclose
 
+from pyomo.common.fileutils import find_library
 from pyomo.common.gc_manager import PauseGC
 from pyomo.opt import ProblemFormat, AbstractProblemWriter, WriterFactory
 from pyomo.core.expr import current as EXPR
@@ -58,6 +59,10 @@ def set_pyomo_amplfunc_env(external_libs):
     env_str = ''
     for _lib in external_libs:
         _lib = _lib.strip()
+        # Convert the library to an absolute path
+        _abs_lib = find_library(_lib)
+        if _abs_lib is not None:
+            _lib = _abs_lib
         if ( ' ' not in _lib
              or ( _lib[0]=='"' and _lib[-1]=='"'
                   and '"' not in _lib[1:-1] )


Test patch:
diff --git a/pyomo/core/tests/unit/test_external.py b/pyomo/core/tests/unit/test_external.py
--- a/pyomo/core/tests/unit/test_external.py
+++ b/pyomo/core/tests/unit/test_external.py
@@ -16,6 +16,7 @@
 import pyomo.common.unittest as unittest
 
 from pyomo.common.getGSL import find_GSL
+from pyomo.common.log import LoggingIntercept
 from pyomo.common.tempfiles import TempfileManager
 from pyomo.environ import (
     ConcreteModel, Block, Var, Objective, Expression, SolverFactory, value,
@@ -450,6 +451,17 @@ def test_load_local_asl_library(self):
             finally:
                 os.chdir(orig_dir)
 
+    def test_unknown_library(self):
+        m = ConcreteModel()
+        with LoggingIntercept() as LOG:
+            m.ef = ExternalFunction(
+                library='unknown_pyomo_external_testing_function',
+                function='f')
+        self.assertEqual(
+            LOG.getvalue(),
+            'Defining AMPL external function, but cannot locate '
+            'specified library "unknown_pyomo_external_testing_function"\n')
+
     def test_eval_gsl_function(self):
         DLL = find_GSL()
         if not DLL: