Repository: Pyomo/pyomo

Issue:
'ConcreteModel' object has no attribute 'as_concrete_model'
rapper throw the following exception!
Traceback (most recent call last):
  File "/home/administrator/project/python/gym-rl/pyomotools/test/pysp/simple_quadratic/run.py", line 10, in <module>
    stsolver = rapper.StochSolver("ReferenceModel.py")
  File "/home/administrator/.conda/envs/python36/lib/python3.6/site-packages/pyomo/pysp/util/rapper.py", line 91, in __init__
    tree_model = tree.as_concrete_model()
  File "/home/administrator/.conda/envs/python36/lib/python3.6/site-packages/pyomo/core/base/block.py", line 519, in __getattr__
    % (self.__class__.__name__, val))
AttributeError: 'ConcreteModel' object has no attribute 'as_concrete_model'



Discussion:

Resolving patch:
diff --git a/pyomo/pysp/util/doc/conf.py b/pyomo/pysp/util/doc/conf.py
deleted file mode 100644
--- a/pyomo/pysp/util/doc/conf.py
+++ /dev/null
@@ -1,193 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# taken from the pyomo documentation build configuration file, created by
-# sphinx-quickstart on Mon Dec 12 16:08:36 2016.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import os
-import sys
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# assumes pyutilib source is next to the pyomo source directory
-sys.path.insert(0, os.path.abspath('../../../pyutilib'))
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = ['sphinx.ext.autodoc',
-              'sphinx.ext.coverage',
-              'sphinx.ext.mathjax',
-              'sphinx.ext.viewcode',
-              'sphinx.ext.napoleon',
-              'sphinx.ext.ifconfig',
-              'sphinx.ext.inheritance_diagram',
-              'sphinx.ext.autosummary',
-              'sphinx.ext.doctest']
-    #'sphinx.ext.githubpages']
-
-viewcode_import = True
-#napoleon_include_private_with_doc = True
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'rapper'
-copyright = u'2018, Sandia National Laboratories'
-author = u'David L. Woodruff'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '0.1'
-# The full version, including alpha/beta/rc tags.
-release = '0.1'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
-numfig = True
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-#html_theme = 'alabaster'
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]     
-    # Override default css to get a larger width for local build
-    def setup(app):
-        app.add_stylesheet('theme_overrides.css')
-    html_context = {
-        'css_files': [
-            '_static/theme_overrides.css',
-        ],
-    }
-else:
-    html_context = {
-        'css_files': [
-            'https://media.readthedocs.org/css/sphinx_rtd_theme.css',
-            'https://media.readthedocs.org/css/readthedocs-doc-embed.css',
-            '_static/theme_overrides.css',
-        ],
-    }
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-
-# -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'rapper'
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'rapper.tex', 'rapper Documentation',
-     'rapper', 'manual'),
-]
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'rapper', 'rapper Documenation',
-     [author], 1)
-]
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'rapper', 'rapper Documentation',
-     author, 'rapper', 'Wrapper for PySP.',
-     'Miscellaneous'),
-]
-
-#autodoc_member_order = 'bysource'
-#autodoc_member_order = 'groupwise'
diff --git a/pyomo/pysp/util/rapper.py b/pyomo/pysp/util/rapper.py
--- a/pyomo/pysp/util/rapper.py
+++ b/pyomo/pysp/util/rapper.py
@@ -5,13 +5,30 @@
 Author: David L. Woodruff, started February 2017
 """
 
+import inspect
 from pyomo.environ import *
 from pyomo.pysp.scenariotree.instance_factory \
     import ScenarioTreeInstanceFactory
-from pyomo.pysp.ef import create_ef_instance
+import pyomo.pysp.ef as pyspef # import (create_ef_instance, solve_ef)
+from pyomo.pysp.ef_writer_script import ExtensiveFormAlgorithm
+from pyomo.pysp.scenariotree.tree_structure_model import CreateAbstractScenarioTreeModel
+from pyomo.pysp.scenariotree.instance_factory import \
+                ScenarioTreeInstanceFactory
+
 import pyomo.pysp.phinit as phinit
 import os
 
+def _optiondict_2_list(phopts, args_list = None):
+    """ A little utility to change the format of options"""
+    if args_list is None:
+        args_list = []
+    if phopts is not None:
+        for key in phopts:
+            args_list.append(key)
+            if phopts[key] is not None:
+                args_list.append(phopts[key])
+    return args_list
+
 def _kwfromphopts(phopts):
     """
     This is really local to the StochSolver __init__ but
@@ -51,17 +68,21 @@ def dointpair(pho, fo):
 
 #==================================
 class StochSolver:
-    """A class for solving stochastic versions of concrete models.
+    """A class for solving stochastic versions of concrete models and
+    abstract models.
     Inspired by the IDAES use case and by daps ability to create tree models.
     Author: David L. Woodruff, February 2017
     
     Args: 
-      fsfile (str): is a file that contains the the scenario callback.
-      fsfct (str): function name in the file, which defaults to
-        "pysp_instance_creation_callback"
-      tree_model: gives the tree as a concrete model.
-        If it is None, then look for a function in fsfile called
-        "pysp_scenario_tree_model_callback" that will return it.
+      fsfile (str): is a path to the file that contains the scenario 
+                    callback for concrete or the reference model for abstract.
+      fsfct (str, or fct, or None): 
+         str   callback function name in the file
+         fct   callback function
+         None  it is a AbstractModel
+      tree_model (concrete model, or path): 
+        gives the tree as a concrete model (which could be a fct)
+        or path to AMPL data file.
       phopts: dictionary of ph options; needed during construction 
               if there is bundling.
 
@@ -70,44 +91,67 @@ class StochSolver:
 
     """
     def __init__(self, fsfile,
-                 fsfct = "pysp_instance_creation_callback",
+                 fsfct = None,
                  tree_model = None,
                  phopts = None):
         """Initialize a StochSolver object.
         """
-        fsfile = fsfile.replace('.py','')  # import does not like .py
-        # __import__ only gives the top level module
-        # probably need to be dealing with modules installed via setup.py
-        m = __import__(fsfile)
-        for n in fsfile.split(".")[1:]:
-            m = getattr(m, n)
-        scen_function = getattr(m, fsfct)
-
-        if tree_model is None:
-            tree_maker = getattr(m, \
-                                 "pysp_scenario_tree_model_callback")
-
-            tree = tree_maker()
-            tree_model = tree.as_concrete_model()
-
-            # DLW March 21: still not correct
-            scenario_instance_factory = \
-                ScenarioTreeInstanceFactory("ReferenceModel.py", tree_model)
-            #ScenarioTreeInstanceFactory(scen_function, tree_model)
+        if fsfct is None:
+            # Changed in October 2018: None implies AbstractModel
+            args_list = _optiondict_2_list(phopts)
+            parser = phinit.construct_ph_options_parser("")
+            options = parser.parse_args(args_list)
 
-        else: 
-            # DLW March 21: still not correct
             scenario_instance_factory = \
-                ScenarioTreeInstanceFactory(scen_function, tree_model)
+                ScenarioTreeInstanceFactory(fsfile, tree_model)
+
+            try:
+                self.scenario_tree = \
+                    phinit.GenerateScenarioTreeForPH(options,
+                                                     scenario_instance_factory)
+            except:
+                print ("ERROR in StochSolver called from",inspect.stack()[1][3])
+                raise RuntimeError("fsfct is None, so assuming",
+                      "AbstractModel but could not find all ingredients.")
+                
+        else:  # concrete model
+            if  callable(fsfct):
+                scen_function = fsfct
+            else: # better be a string
+                fsfile = fsfile.replace('.py','')  # import does not like .py
+                # __import__ only gives the top level module
+                # probably need to be dealing with modules installed via setup.py
+                m = __import__(fsfile)
+                for n in fsfile.split(".")[1:]:
+                    m = getattr(m, n)
+                scen_function = getattr(m, fsfct)
+
+            if tree_model is None:
+                treecbname = "pysp_scenario_tree_model_callback"
+                tree_maker = getattr(m, treecbname)
+
+                tree = tree_maker()
+                if isinstance(tree, Pyo.ConcreteModel):
+                    tree_model = tree
+                else:
+                    raise RuntimeError("The tree returned by",treecbname,
+                                       "must be a ConcreteModel") 
+                    
+                scenario_instance_factory = ScenarioTreeInstanceFactory(scen_function, tree_model)
+
+            else: 
+                # DLW March 21: still not correct
+                scenario_instance_factory = \
+                    ScenarioTreeInstanceFactory(scen_function, tree_model)
+
 
+            kwargs = _kwfromphopts(phopts)
+            self.scenario_tree = \
+                scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
+            instances = scenario_instance_factory. \
+                        construct_instances_for_scenario_tree(self.scenario_tree)
+            self.scenario_tree.linkInInstances(instances)        
 
-        kwargs = _kwfromphopts(phopts)
-        self.scenario_tree = \
-            scenario_instance_factory.generate_scenario_tree(**kwargs) #verbose = True)
-        instances = scenario_instance_factory. \
-                    construct_instances_for_scenario_tree(self.scenario_tree)
-        self.scenario_tree.linkInInstances(instances)        
- 
     #=========================
     def make_ef(self, verbose=False):
         """ Make an ef object (used by solve_ef)
@@ -118,7 +162,7 @@ def make_ef(self, verbose=False):
         Returns:
             ef_instance: the ef object
         """
-        return create_ef_instance(self.scenario_tree, verbose_output=verbose)
+        return pyspef.create_ef_instance(self.scenario_tree, verbose_output=verbose)
     
     def solve_ef(self, subsolver, sopts = None, tee = False, need_gap = False):
         """Solve the stochastic program directly using the extensive form.
@@ -195,11 +239,7 @@ def solve_ph(self, subsolver, default_rho, phopts = None, sopts = None):
         phargslist = ['--default-rho',str(default_rho)]
         phargslist.append('--solver')
         phargslist.append(str(subsolver))
-        if phopts is not None:
-            for key in phopts:
-                phargslist.append(key)
-                if phopts[key] is not None:
-                    phargslist.append(phopts[key])
+        phargslist = _optiondict_2_list(phopts, args_list = phargslist)
                     
         # Subproblem options go to PH as space-delimited, equals-separated pairs.
         if sopts is not None:


Test patch:
diff --git a/pyomo/pysp/tests/rapper/__init__.py b/pyomo/pysp/tests/rapper/__init__.py
new file mode 100644
--- /dev/null
+++ b/pyomo/pysp/tests/rapper/__init__.py
@@ -0,0 +1,12 @@
+#  ___________________________________________________________________________
+#
+#  Pyomo: Python Optimization Modeling Objects
+#  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
+#  Under the terms of Contract DE-NA0003525 with National Technology and 
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  rights in this software.
+#  This software is distributed under the 3-clause BSD License.
+#  ___________________________________________________________________________
+#
+# Unit tests for PySP wrapper called rapper
+#
diff --git a/pyomo/pysp/tests/rapper/abstract_rapper_tester.py b/pyomo/pysp/tests/rapper/abstract_rapper_tester.py
new file mode 100644
--- /dev/null
+++ b/pyomo/pysp/tests/rapper/abstract_rapper_tester.py
@@ -0,0 +1,71 @@
+# Provide some test for rapper
+# Author: David L. Woodruff (Sept 2018)
+
+import pyutilib.th as unittest
+import tempfile
+import sys
+import os
+import shutil
+import json
+import pyomo.environ as pyo
+import pyomo.pysp.util.rapper as rapper
+from pyomo.pysp.scenariotree.tree_structure_model import CreateAbstractScenarioTreeModel
+import pyomo as pyomoroot
+
+__author__ = 'David L. Woodruff <DLWoodruff@UCDavis.edu>'
+__version__ = 1.5
+
+solvername = "ipopt" # could use almost any solver
+
+class Test_abstract_rapper(unittest.TestCase):
+    """ Test the rapper code."""
+
+    def setUp(self):
+        """ Get ready for tests"""
+
+        p = str(pyomoroot.__path__)
+        l = p.find("'")
+        r = p.find("'", l+1)
+        pyomorootpath = p[l+1:r]
+        farmpath = pyomorootpath + os.sep + ".." + os.sep + "examples" + \
+                   os.sep + "pysp" + os.sep + "farmer"
+        farmpath = os.path.abspath(farmpath)
+        
+        # for AbstractModels
+        self.farmer_ReferencePath = farmpath + os.sep + \
+                                    "models" + os.sep + "ReferenceModel.py"
+        self.farmer_scenarioPath = farmpath + os.sep + \
+                                    "scenariodata"
+        
+    def tearDown(self):
+        # from GH: This step is key, as Python keys off the name of the module, not the location.
+        #       So, different reference models in different directories won't be detected.
+        #       If you don't do this, the symptom is a model that doesn't have the attributes
+        #       that the data file expects.
+        if "ReferenceModel" in sys.modules:
+            del sys.modules["ReferenceModel"]
+
+
+
+    def test_Abstract_Construction(self):
+        """ see if we can create the solver object for an AbstractModel"""
+             
+        stsolver = rapper.StochSolver(self.farmer_ReferencePath,
+                                      fsfct = None,
+                                      tree_model = self.farmer_scenarioPath,
+                                      phopts = None)
+        
+    def test_Abstract_ef(self):
+        """ see if we can create the solver object for an AbstractModel"""
+             
+        stsolver = rapper.StochSolver(self.farmer_ReferencePath,
+                                      fsfct = None,
+                                      tree_model = self.farmer_scenarioPath,
+                                      phopts = None)
+        ef_sol = stsolver.solve_ef(solvername)
+        assert(ef_sol.solver.termination_condition \
+               ==  pyo.TerminationCondition.optimal)
+
+# see also foo.py        
+if __name__ == '__main__':
+    unittest.main()
diff --git a/pyomo/pysp/tests/rapper/rapper_tester.py b/pyomo/pysp/tests/rapper/rapper_tester.py
--- a/pyomo/pysp/tests/rapper/rapper_tester.py
+++ b/pyomo/pysp/tests/rapper/rapper_tester.py
@@ -1,7 +1,7 @@
 # Provide some test for rapper
 # Author: David L. Woodruff (circa March 2017 and Sept 2018)
 
-import unittest
+import pyutilib.th as unittest
 import tempfile
 import sys
 import os
@@ -16,7 +16,7 @@
 __date__ = 'August 14, 2017'
 __version__ = 1.4
 
-solvername = "cplex" # could use almost any solver
+solvername = "ipopt" # could use almost any solver
 
 class Testrapper(unittest.TestCase):
     """ Test the rapper code."""
@@ -30,6 +30,7 @@ def setUp(self):
 
         """ During debugging, local files might get in the way
         of finding the file in the temp dir, so we cd there."""
+        self.savecwd = os.getcwd()
         os.chdir(self.tdir)
 
         p = str(pyomoroot.__path__)
@@ -51,13 +52,42 @@ def setUp(self):
                         self.tdir + os.sep + "ScenarioStructure.dat")
         self.farmer_concrete_tree = \
                 abstract_tree.create_instance("ScenarioStructure.dat")
-        
+
     def tearDown(self):
-        pass
+        # from GH: This step is key, as Python keys off the name of the module, not the location.
+        #       So, different reference models in different directories won't be detected.
+        #       If you don't do this, the symptom is a model that doesn't have the attributes
+        #       that the data file expects.
+        if "ReferenceModel" in sys.modules:
+            del sys.modules["ReferenceModel"]
+
+        os.chdir(self.savecwd)
+
+    def test_fct_contruct(self):
+        """ give a callback function rather than a string"""
+        from ReferenceModel import pysp_instance_creation_callback
+        stsolver = rapper.StochSolver(None,
+                                fsfct = pysp_instance_creation_callback,
+                                tree_model = self.farmer_concrete_tree)
+
+    def test_no_fsfct_no_tree(self):
+        """verify that deprecated concrete with no fsfct is an error"""
+        with self.assertRaises(RuntimeError):
+            stsolver = rapper.StochSolver("ReferenceModel.py",
+                                          fsfct = None,
+                                          tree_model = None)
+
+    def test_construct_default_tree_error(self):
+        """verify that construction of concrete with default tree name gives error when it should"""
+        with self.assertRaises(AttributeError):
+            stsolver = rapper.StochSolver("ReferenceModel.py",
+                                          fsfct = "pysp_instance_creation_callback",
+                                          tree_model = None)
 
     def test_ef_solve(self):
         """ solve the ef and check some post solution code"""
         stsolver = rapper.StochSolver("ReferenceModel.py",
+                                      fsfct = "pysp_instance_creation_callback",
                                 tree_model = self.farmer_concrete_tree)
         ef_sol = stsolver.solve_ef(solvername)
         assert(ef_sol.solver.termination_condition \
@@ -71,14 +101,16 @@ def test_ef_solve(self):
     def test_ef_solve_with_gap(self):
         """ solve the ef and report gap"""
         stsolver = rapper.StochSolver("ReferenceModel.py",
+                                      fsfct = "pysp_instance_creation_callback",
                                 tree_model = self.farmer_concrete_tree)
         res, gap = stsolver.solve_ef(solvername, tee=True, need_gap=True)
 
     def test_ph_solve(self):
-        """ use ph; assumes concrete two-stage json passes"""
+        """ use ph"""
         phopts = {'--max-iterations': '2'}
         stsolver = rapper.StochSolver("ReferenceModel.py",
                                       tree_model = self.farmer_concrete_tree,
+                                      fsfct = "pysp_instance_creation_callback",
                                       phopts = phopts)
         ph = stsolver.solve_ph(subsolver = solvername, default_rho = 1,
                                phopts=phopts)
@@ -90,5 +122,6 @@ def test_ph_solve(self):
             pass
         assert(nodename == 'RootNode')
 
+# see also foo.py        
 if __name__ == '__main__':
     unittest.main()