Repository: Pyomo/pyomo

Issue:
pyomo/contrib/parmest/ Invalid index error for component
Hi

I'm trying to estimate the parameter 'Boiler[1].eta' with paramest.py. My theta_names looks like this: theta_names = ['Boiler[1].eta'].

Unfortunately i get this error message:
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pyomo/pysp/phutils.py", line 482, in extractComponentIndices
    % (str(tuple(index_template)), component.name))
ValueError: The index ('1].eta',) is not valid for component named: Boiler

If i pause the program at the end of the execution of my model_function and execute the following line:
model.Boiler[1].eta 
I get this: 
<pyomo.core.base.var.SimpleVar at 0x12090e748>
So the variable actually exists in my model.

Is this a Bug or did i do something wrong? 
The index ('1].eta',) looks also a bit chopped up. Maybe there is a Problem?

Greetings
Jakob


Discussion:
It may take a few days to look into this.

On Thu, Mar 12, 2020 at 3:55 PM Jakob <notifications@github.com> wrote:

> Hi
>
> I'm trying to estimate the parameter 'Boiler[1].eta' with paramest.py. My
> theta_names looks like this: theta_names = ['Boiler[1].eta'].
>
> Unfortunately i get this error message:
> File
> "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pyomo/pysp/phutils.py",
> line 482, in extractComponentIndices
> % (str(tuple(index_template)), component.name))
> ValueError: The index ('1].eta',) is not valid for component named: Boiler
>
> If i pause the program at the end of the execution of my model_function
> and execute the following line:
> model.Boiler[1].eta
> I get this:
> <pyomo.core.base.var.SimpleVar at 0x12090e748>
> So the variable actually exists in my model.
>
> Is this a Bug or did i do something wrong?
> The index ('1].eta',) looks also a bit chopped up. Maybe there is a
> Problem?
>
> Greetings
> Jakob
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/Pyomo/pyomo/issues/1340>, or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AD4VTBAVCM6PU6LNWCIH7LLRHFR5HANCNFSM4LGWZXCA>
> .
>

Okay! Thank you :)
May i ask how long this will approximately take to fix this bug? 
I had a similar problem. Maybe it has something to do with the block architecture of the model
I created a minimal example for this issue!
```
from matplotlib.pyplot import plot
import matplotlib.pyplot as plt
import pandas as pd

from pyomo.environ import *
import pyomo.contrib.parmest.parmest as parmest


def model_function(data):
    model = AbstractModel()

    model.T = Set(ordered=True)

    model.J = Set(initialize=[1])

    instance = model.create_instance(data={None: {
        'T': {None: [0, 1, 2, 3], 'J': [1]}
    }})

    def boiler_rule(A, u):
        A.eta = Var(bounds=(0, 1), initialize=0.5)
        A.eta.fix()
        A.cost_per_kWh = Param(initialize=1)

        A.P_in = Var(instance.T, within=NonNegativeReals)
        A.P_out = Var(instance.T, within=NonNegativeReals)

        # constraint efficiancy
        def eta_constraint_rule(A, t):
            return A.P_out[t] == A.P_in[t] * A.eta

        A.constraint_eta = Constraint(instance.T, rule=eta_constraint_rule)

    instance.Boiler = Block(instance.J, rule=boiler_rule)
    instance.Boiler.construct()

    instance.demand = Param(instance.T, initialize={0: 0, 1: 600, 2: 900, 3: 1200})
    instance.cost_error = Param(initialize=100)

    def Bilanz_Regel(instance, t):
        return instance.demand[t] == instance.Boiler[1].P_out[t]

    instance.balance = Constraint(instance.T, rule=Bilanz_Regel)

    instance.Zielfunktion = Objective(expr=sum(
        instance.Boiler[1].P_in[t] * instance.Boiler[1].cost_per_kWh * instance.cost_error for t in instance.T))
    return instance


def parameter_estimation():
    """Estimate Parameter Boiler[1].eta"""
    data = {'T': [0, 1, 2, 3],
            'P_in': {0: 0.0, 1: 1200.0, 2: 1800.0, 3: 2400.0},
            'P_out': {0: 0.0, 1: 600.0, 2: 900.0, 3: 1200.0}}
    data = [data]

    # theta_names
    # The third argument is a list of variable names that the user wants to estimate.
    # The list contains strings with Var names from the Pyomo model.
    theta_names = ['Boiler[1].eta']

    # objective_function
    def objective_function(model, data):
        expr = 0
        for t in data['T']:
            # (observed_value - value_calculated_form_model)^2*1/nr_time_steps
            # Achtung: Geht davon aus, dass alle Zeitschritte gleich lang sind.
            # Muss für variable Schrittweite ggf. angepasst werden.
            # P_in
            expr += ((data['P_in'][t] - model.Boiler[1].P_in[t]) ** 2) * (1 / len(data['T']))
            # P_out
            expr += ((data['P_in'][t] - model.Boiler[1].P_out[t]) ** 2) * (1 / len(data['T']))
        return expr

    pest = parmest.Estimator(model_function, data, theta_names, objective_function)
    obj, theta = pest.theta_est()


def solve_model():
    """this represents a normal solve of the model, without parameter estimation."""
    instance = model_function()
    # solver
    result = SolverFactory('glpk').solve(instance, tee=True)
    print(result)

    # plotte Verbrauch, den die Anlagen erfüllen müssen
    plt.figure(1)
    plot(pd.Series(instance.demand.extract_values()), drawstyle='steps-post')
    plt.show();
    print('Demand')

    # plotte die bereitgestellte Leistung P_out der Anlagen 1(blau) und 2(orange)
    # der Verbrauch ist noch mal gepunktet dargestellt
    plt.figure(2)
    plot(pd.Series(instance.Boiler[1].P_out.extract_values()), drawstyle='steps-post')

    plot(pd.Series(instance.demand.extract_values()), 'k:', drawstyle='steps-post')
    plt.legend(['Boiler 1', 'demand'])
    plt.show();

    print(instance.Boiler[1].P_in.extract_values())
    print(instance.Boiler[1].P_out.extract_values())


# solve_model([])
parameter_estimation()
```
This particular error message is created by Pyomo and has to do with how you are trying to create a ConcreteModel starting from an abstract model. It is not a bug in parmest. In this case, I recommend changing to a ConcreteModel from the start, e.g.,
def model_function():
    instance= ConcreteModel()
    instance.T = Set(ordered=True, initialize=[0,1,2,3])
    instance.J = Set(initialize=[1])

After you fix that, there are a lot of other troubles (e.g., parmest wants to pass in the data, but you have it all hard-wired). If you need help, you should write to the Pyomo forum or send me an email directly, rather than opening an issue, though.
OK. There is a bug in Parmest concerning Vars on blocks and I want to do a major revision to take
care of it. I will start on it today, but it will take a few days.

Resolving patch:
diff --git a/pyomo/contrib/parmest/parmest.py b/pyomo/contrib/parmest/parmest.py
--- a/pyomo/contrib/parmest/parmest.py
+++ b/pyomo/contrib/parmest/parmest.py
@@ -22,6 +22,7 @@
 
 import pyomo.environ as pyo
 import pyomo.pysp.util.rapper as st
+from pyomo.pysp.scenariotree import tree_structure
 from pyomo.pysp.scenariotree.tree_structure_model import CreateAbstractScenarioTreeModel
 from pyomo.opt import SolverFactory
 from pyomo.environ import Block
@@ -463,9 +464,21 @@ def _Q_opt(self, ThetaVals=None, solver="ef_ipopt",
             tree_model.BootList = bootlist
         tree_model.cb_data = self.callback_data  # None is OK
 
-        stsolver = st.StochSolver(fsfile = "pyomo.contrib.parmest.parmest",
-                                  fsfct = "_pysp_instance_creation_callback",
-                                  tree_model = tree_model)
+        try:
+            # For structured models, it is important that we use the
+            # updated version of the CUID representation.  PySP (for
+            # backwards compatibility reasons, and so a ton of tests
+            # don't have to be updated) still defaults to the old
+            # representation.
+            _cuidver = tree_structure.CUID_repr_version
+            tree_structure.CUID_repr_version = 2
+            stsolver = st.StochSolver(
+                fsfile = "pyomo.contrib.parmest.parmest",
+                fsfct = "_pysp_instance_creation_callback",
+                tree_model = tree_model
+            )
+        finally:
+            tree_structure.CUID_repr_version = _cuidver
                 
         # Solve the extensive form with ipopt
         if solver == "ef_ipopt":
diff --git a/pyomo/pysp/embeddedsp.py b/pyomo/pysp/embeddedsp.py
--- a/pyomo/pysp/embeddedsp.py
+++ b/pyomo/pysp/embeddedsp.py
@@ -34,6 +34,7 @@
                                     StochasticConstraintBodyAnnotation,
                                     StochasticObjectiveAnnotation,
                                     StochasticVariableBoundsAnnotation)
+from pyomo.pysp.scenariotree import tree_structure
 from pyomo.pysp.scenariotree.tree_structure_model import \
     CreateAbstractScenarioTreeModel
 from pyomo.pysp.scenariotree.manager import \
@@ -399,7 +400,8 @@ def __init__(self, reference_model):
         self.time_stages = tuple(sorted(self.stage_to_variables_map))
         assert self.time_stages[0] == 1
         self.variable_symbols = ComponentUID.generate_cuid_string_map(
-            self.reference_model, ctype=Var, repr_version=1)
+            self.reference_model, ctype=Var,
+            repr_version=tree_structure.CUID_repr_version)
         # remove the parent blocks from this map
         keys_to_delete = []
         for var in self.variable_symbols:
diff --git a/pyomo/pysp/scenariotree/tree_structure.py b/pyomo/pysp/scenariotree/tree_structure.py
--- a/pyomo/pysp/scenariotree/tree_structure.py
+++ b/pyomo/pysp/scenariotree/tree_structure.py
@@ -44,13 +44,16 @@
 
 logger = logging.getLogger('pyomo.pysp')
 
+CUID_repr_version = 1
+
 class _CUIDLabeler(object):
     def __init__(self):
         self._cuid_map = ComponentMap()
 
     def update_cache(self, block):
         self._cuid_map.update(
-            ComponentUID.generate_cuid_string_map(block, repr_version=1))
+            ComponentUID.generate_cuid_string_map(
+                block, repr_version=CUID_repr_version))
 
     def clear_cache(self):
         self._cuid_map = {}


Test patch: