Repository: Pyomo/pyomo

Issue:
NameError: name 'SolverFactory' is not defined
I try to run a (dummy) model on the Neos server by executing the following code in Python 3.7:

```!python
from pyomo import environ as pyo  # type: ignore

model = pyo.ConcreteModel()
model.x = pyo.Var([1, 2], domain=pyo.NonNegativeReals)
model.OBJ = pyo.Objective(expr=2*model.x[1] + 3*model.x[2])
model.Constraint1 = pyo.Constraint(expr=3*model.x[1] + 4*model.x[2] >= 1)

solver_manager = pyo.SolverManagerFactory('neos')
results = solver_manager.solve(model, opt='cplex')
results.write()
```

but then I get the following error:
```
Traceback (most recent call last):
  File "./src/opt.py", line 9, in <module>
    results = solver_manager.solve(model, opt='cplex')
  File "[..]site-packages/pyomo/opt/parallel/async_solver.py", line 28, in solve
    return self.execute(*args, **kwds)
  File "[..]site-packages/pyomo/opt/parallel/manager.py", line 107, in execute
    ah = self.queue(*args, **kwds)
  File "[..]site-packages/pyomo/opt/parallel/manager.py", line 122, in queue
    return self._perform_queue(ah, *args, **kwds)
  File "[..]site-packages/pyomo/neos/plugins/kestrel_plugin.py", line 111, in _perform_queue
    opt = SolverFactory('_neos')
NameError: name 'SolverFactory' is not defined
```

I tried with other solvers than cplex as well, but that does not seem to be the issue.


Discussion:

Resolving patch:
diff --git a/pyomo/neos/kestrel.py b/pyomo/neos/kestrel.py
old mode 100755
new mode 100644
--- a/pyomo/neos/kestrel.py
+++ b/pyomo/neos/kestrel.py
@@ -26,6 +26,8 @@
 import tempfile
 import logging
 
+logger = logging.getLogger('pyomo.neos')
+
 class NEOS(object):
     # NEOS currently only supports HTTPS access
     scheme = 'https'
@@ -104,8 +106,6 @@ def make_connection(self, host):
             connection.set_tunnel(host)
             return connection
 
-logger = logging.getLogger('pyomo.solvers')
-
 
 class kestrelAMPL:
 
@@ -147,7 +147,7 @@ def tempfile(self):
 
     def kill(self,jobnumber,password):
         response = self.neos.killJob(jobNumber,password)
-        sys.stdout.write(response+"\n")
+        logger.info(response)
 
     def solvers(self):
         return self.neos.listSolversInCategory("kestrel") \
@@ -168,15 +168,27 @@ def retrieve(self,stub,jobNumber,password):
         solfile.close()
 
     def submit(self, xml):
-        user = "%s on %s" % (os.getenv('LOGNAME'),socket.getfqdn(socket.gethostname()))
+        # LOGNAME and USER should map to the effective user (i.e., the
+        # one sudo-ed to), whereas USERNAME is the original user who ran
+        # sudo.  We include USERNAME to cover Windows, where LOGNAME and
+        # USER may not be defined.
+        for _ in ('LOGNAME','USER','USERNAME'):
+            uname = os.getenv(_)
+            if uname is not None:
+                break
+        hostname = socket.getfqdn(socket.gethostname())
+        user = "%s on %s" % (uname,hostname)
         (jobNumber,password) = self.neos.submitJob(xml,user,"kestrel")
         if jobNumber == 0:
-            sys.stdout.write("Error: %s\nJob not submitted.\n" % password)
-            sys.exit(1)
-        sys.stdout.write("Job %d submitted to NEOS, password='%s'\n" %
-                         (jobNumber,password))
-        sys.stdout.write("Check the following URL for progress report :\n")
-        sys.stdout.write(urlscheme+"://www.neos-server.org/neos/cgi-bin/nph-neos-solver.cgi?admin=results&jobnumber=%d&pass=%s\n" % (jobNumber,password))
+            raise RuntimeError("%s\n\tJob not submitted" % (password,))
+
+        logger.info("Job %d submitted to NEOS, password='%s'\n" %
+                    (jobNumber,password))
+        logger.info("Check the following URL for progress report :\n")
+        logger.info(
+            "%s://www.neos-server.org/neos/cgi-bin/nph-neos-solver.cgi"
+            "?admin=results&jobnumber=%d&pass=%s\n"
+            % (NEOS.scheme, jobNumber,password))
         return (jobNumber,password)
 
     def getJobAndPassword(self):
@@ -228,18 +240,14 @@ def getSolverName(self):
                       break
                 #
                 if not NEOS_solver_name:
-                    sys.stdout.write("%s is not available on NEOS.  Choose from:\n" % solver_name)
-                    for s in kestrelAmplSolvers:
-                        sys.stdout.write("\t%s\n"%s)
-                    sys.stdout.write('To choose: option kestrel_options "solver=xxx";\n\n')
-                    sys.exit(1)
+                    raise RuntimeError(
+                        "%s is not available on NEOS.  Choose from:\n\t%s"
+                        % (solver_name, "\n\t".join(kestrelAmplSolvers)))
         #
         if self.options is None or m is None:
-            sys.stdout.write("No solver name selected.  Choose from:\n")
-            for s in kestrelAmplSolvers:
-                sys.stdout.write("\t%s\n"%s)
-            sys.stdout.write('\nTo choose: option kestrel_options "solver=xxx";\n\n')
-            sys.exit(1)
+            raise RuntimeError(
+                "%s is not available on NEOS.  Choose from:\n\t%s"
+                % (solver_name, "\n\t".join(kestrelAmplSolvers)))
         return NEOS_solver_name
 
     def formXML(self,stub):
@@ -377,7 +385,7 @@ def formXML(self,stub):
       if not jobNumber:
         xml = kestrel.formXML(stub)
         (jobNumber,password) = kestrel.submit(xml)
-      
+
     except KeyboardInterrupt:
       e = sys.exc_info()[1]
       sys.stdout.write("Keyboard Interrupt while submitting problem.\n")
@@ -390,7 +398,7 @@ def formXML(self,stub):
       while status == "Running" or status == "Waiting":
         (output,offset) = kestrel.neos.getIntermediateResults(jobNumber,
                                                            password,offset)
-      
+
         if isinstance(output,xmlrpclib.Binary):
           output = output.data
         sys.stdout.write(output)
diff --git a/pyomo/neos/plugins/NEOS.py b/pyomo/neos/plugins/NEOS.py
--- a/pyomo/neos/plugins/NEOS.py
+++ b/pyomo/neos/plugins/NEOS.py
@@ -2,12 +2,14 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 
+import logging
+
 import pyutilib.misc
 import pyutilib.services
 
@@ -15,6 +17,8 @@
 from pyomo.opt.results import *
 from pyomo.opt.solver import *
 
+logger = logging.getLogger('pyomo.neos')
+
 @SolverFactory.register('_neos', 'Interface for solvers hosted on NEOS')
 class NEOSRemoteSolver(SystemCallSolver):
     """A wrapper class for NEOS Remote Solvers"""
@@ -45,11 +49,11 @@ def create_command_line(self, executable, problem_files):
         # in case something crashes unexpectedly, which is not without precedent.
         if self._keepfiles:
             if self._log_file is not None:
-                print("Solver log file: '%s'" % self._log_file)
+                logger.info("Solver log file: '%s'" % (self._log_file,))
             if self._soln_file is not None:
-                print("Solver solution file: '%s'" % self._soln_file)
+                logger.info("Solver solution file: '%s'" % (self._soln_file,))
             if self._problem_files is not []:
-                print("Solver problem files: %s" % str(self._problem_files))
+                logger.info("Solver problem files: %s" % (self._problem_files,))
 
         return pyutilib.misc.Bunch(cmd="", log_file=self._log_file, env="")
 
diff --git a/pyomo/neos/plugins/kestrel_plugin.py b/pyomo/neos/plugins/kestrel_plugin.py
--- a/pyomo/neos/plugins/kestrel_plugin.py
+++ b/pyomo/neos/plugins/kestrel_plugin.py
@@ -2,31 +2,35 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 
+import logging
 import os
 import re
 import six
 
 from six.moves.xmlrpc_client import ProtocolError
 
-from pyomo.opt.parallel.manager import *
-from pyomo.opt.parallel.async_solver import *
+from pyomo.opt import SolverFactory, SolverManagerFactory, OptSolver
+from pyomo.opt.parallel.async_solver import (
+    AsynchronousSolverManager, ActionStatus
+)
 from pyomo.opt.base import OptSolver
 from pyomo.core.base import Block
 import pyomo.neos.kestrel
 
-import logging
+
 logger = logging.getLogger('pyomo.neos')
 
-error_re = re.compile('error', flags=re.I)
-warn_re = re.compile('warn', flags=re.I)
 
 def _neos_error(msg, results, current_message):
+    error_re = re.compile('error', flags=re.I)
+    warn_re = re.compile('warn', flags=re.I)
+
     logger.error("%s  NEOS log:\n%s" % ( msg, current_message, ))
     soln_data = results.data
     if six.PY3:
@@ -38,7 +42,8 @@ def _neos_error(msg, results, current_message):
             logger.warn(line)
 
 
-@SolverManagerFactory.register('neos', doc="Asynchronously execute solvers on the NEOS server")
+@SolverManagerFactory.register(
+    'neos', doc="Asynchronously execute solvers on the NEOS server")
 class SolverManager_NEOS(AsynchronousSolverManager):
 
     def clear(self):
@@ -47,7 +52,8 @@ def clear(self):
         """
         AsynchronousSolverManager.clear(self)
         self.kestrel = pyomo.neos.kestrel.kestrelAMPL()
-        self._ah = {} # maps NEOS job numbers to their corresponding action handle.
+        self._ah = {} # maps NEOS job numbers to their corresponding
+                      # action handle.
         self._args = {}
         self._opt_data = {}
 
@@ -131,7 +137,7 @@ def _perform_queue(self, ah, *args, **kwds):
         os.environ['kestrel_options'] = 'solver=%s' % self._solvers[solver_name]
         #
         # Set the <solver>_options environment
-        # 
+        #
         solver_options = {}
         for key in opt.options:
             solver_options[key]=opt.options[key]
@@ -201,12 +207,12 @@ def _perform_wait_any(self):
 
                 (current_offset, current_message) = self._neos_log[jobNumber]
                 with open(opt._log_file, 'w') as OUTPUT:
-                    six.print_(current_message, file=OUTPUT)
+                    OUTPUT.write(current_message)
                 with open(opt._soln_file, 'w') as OUTPUT:
                     if six.PY2:
-                        six.print_(results.data, file=OUTPUT)
+                        OUTPUT.write(results.data)
                     else:
-                        six.print_((results.data).decode('utf-8'), file=OUTPUT)
+                        OUTPUT.write(results.data.decode('utf-8'))
 
                 rc = None
                 try:
@@ -218,7 +224,6 @@ def _perform_wait_any(self):
 
                 solver_results._smap_id = smap_id
                 self.results[ah.id] = solver_results
-                opt.deactivate()
 
                 if isinstance(args[0], Block):
                     _model = args[0]
@@ -258,7 +263,7 @@ def _perform_wait_any(self):
                             jobNumber,
                             self._ah[jobNumber].password,
                             current_offset )
-                    six.print_(message_fragment, end="")
+                    logger.info(message_fragment)
                     self._neos_log[jobNumber] = (
                         new_offset,
                         current_message + (


Test patch:
diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py
--- a/pyomo/neos/tests/test_neos.py
+++ b/pyomo/neos/tests/test_neos.py
@@ -2,8 +2,8 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
@@ -21,17 +21,18 @@
 from pyomo.scripting.util import cleanup
 from pyomo.neos.kestrel import kestrelAMPL
 
+import pyomo.environ as pyo
+
 from six import iteritems
 
+neos_available = False
 try:
-    kestrel = kestrelAMPL()
+    # Attempt a connection to NEOS.  Any failure will result in skipping
+    # these tests
+    if kestrelAMPL().neos is not None:
+        neos_available = True
 except:
-    kestrel = None
-if kestrel is None or kestrel.neos is None:
-    using_neos = False
-else:
-    using_neos = True
-kestrel = None
+    pass
 
 try:
     import yaml
@@ -40,97 +41,56 @@
     yaml_available=False
 
 
-class CommonTests(object):
-
-    @classmethod
-    def setUpClass(cls):
-        import pyomo.environ
-
-    def setUp(self):
-        try:
-            os.remove(join(currdir,'result.yml'))
-        except OSError:
-            pass
-
-    def pyomo(self, *args, **kwds):
-        args=list(args)
-        args.append('--solver-manager=neos')
-        args.append('-c')
-        if 'solver' in kwds:
-            args.append('--solver='+kwds['solver'])
-        args.append('--symbolic-solver-labels')
-        args.append('--save-results=result.yml')
-        #args.append('--tempdir='+currdir)
-        #args.append('--keepfiles')
-        #args.append('--debug')
-        #args.append('--verbose')
-        os.chdir(currdir)
-
-        print('***')
-        print(' '.join(args))
+#
+# Because the Kestrel tests require connections to the NEOS server, and
+# that can take quite a while (5-20+ seconds), we will only run these
+# tests as part of the nightly suite (i.e., by the CI system as part of
+# PR / master tests)
+#
+@unittest.category('nightly')
+@unittest.skipIf(not neos_available, "Cannot make connection to NEOS server")
+class TestKestrel(unittest.TestCase):
+
+    @unittest.skipIf(not yaml_available, "YAML is not available")
+    def test_pyomo_command(self):
+        results = os.path.join(currdir, 'result.yml')
+        args = [
+            os.path.join(currdir,'t1.py'),
+            '--solver-manager=neos',
+            '--solver=cbc',
+            '--symbolic-solver-labels',
+            '--save-results=%s' % results,
+            '-c',
+            ]
         try:
             output = main.run(args)
-        except:
-            output = None
-        cleanup()
-        print('***')
-        return output
-
-    def check(self, problem, solver):
-        pass
-
-
-class Solver(unittest.TestCase):
-
-    def referenceFile(self, problem, solver):
-        return join(currdir, problem+'.txt')
-
-    def getObjective(self, fname):
-        FILE = open(fname)
-        data = yaml.load(FILE)
-        FILE.close()
-        solutions = data.get('Solution', [])
-        ans = []
-        for x in solutions:
-            ans.append(x.get('Objective', {}))
-        return ans
-
-    def check(self, problem, solver):
-        refObj = self.getObjective(self.referenceFile(problem,solver))
-        ansObj = self.getObjective(join(currdir,'result.yml'))
-        self.assertEqual(len(refObj), len(ansObj))
-        for i in range(len(refObj)):
-            self.assertEqual(len(refObj[i]), len(ansObj[i]))
-            for key,val in iteritems(refObj[i]):
-                self.assertEqual(val, ansObj[i].get(key,None))
-
-
-@unittest.category('expensive')
-@unittest.skipIf(not yaml_available, "YAML is not available")
-@unittest.skipIf(not using_neos, "NEOS is not available")
-class Solve_CBC(Solver, CommonTests):
-
-    def pyomo(self,  *args, **kwds):
-        kwds['solver'] = 'cbc'
-        return CommonTests.pyomo(self, *args, **kwds)
-
-    def test_t1(self):
-        self.problem='test_t1'
-        self.pyomo( join(currdir,'t1.py') )
-        self.check( 't1', 'linear_dual' )
-
-
-@unittest.category('expensive')
-@unittest.skipIf(not using_neos, "NEOS is not available")
-class Misc(unittest.TestCase, CommonTests):
-
-    def test_bad_solver(self):
-        self.problem='test_t1'
-        ans = self.pyomo( join(currdir,'t1.py'), solver='foo')
-        if not ans is None:
-            self.fail("Expected failure because solver is not defined.")
-
-
+            self.assertEqual(output.errorcode, 0)
+
+            with open(results) as FILE:
+                data = yaml.load(FILE)
+            self.assertEqual(
+                data['Solver'][0]['Status'], 'ok')
+            self.assertAlmostEqual(
+                data['Solution'][1]['Status'], 'optimal')
+            self.assertAlmostEqual(
+                data['Solution'][1]['Objective']['o']['Value'], 1)
+            self.assertAlmostEqual(
+                data['Solution'][1]['Variable']['x']['Value'], 0.5)
+        finally:
+            cleanup()
+            os.remove(results)
+
+    def test_kestrel_plugin(self):
+        m = pyo.ConcreteModel()
+        m.x = pyo.Var(bounds=(0,1), initialize=0)
+        m.c = pyo.Constraint(expr=m.x <= 0.5)
+        m.obj = pyo.Objective(expr=2*m.x, sense=pyo.maximize)
+
+        solver_manager = pyo.SolverManagerFactory('neos')
+        results = solver_manager.solve(m, opt='cbc')
+
+        self.assertEqual(results.solver[0].status, pyo.SolverStatus.ok)
+        self.assertAlmostEqual(pyo.value(m.x), 0.5)
 
 
 if __name__ == "__main__":