Repository: Pyomo/pyomo

Issue:
pyomo.gdp fails to transform nested Disjunctions when the parent Disjunction is specified as the target
## Summary

If you specify a Disjunction (SimpleDisjunction or DisjunctionData) as a target of a `pyomo.gdp` transformation, we never touch `_transform_blockData`, which is what we use to start transforming from the inside out. As a result, we panic when we encounter an active `Disjunct`, even though it is our fault for not first descending into the Disjuncts of the target...

### Steps to reproduce the issue

```
from pyomo.environ import *
from pyomo.gdp import *

m = ConcreteModel()
m.x = Var(bounds=(0, 10))
m.y = Var(bounds=(0, 20))

m.disj1 = Disjunct()
m.disj1.disj1 = Disjunct()
m.disj1.disj1.c = Constraint(expr=m.x + m.y <= 18)
m.disj1.disj2 = Disjunct()
m.disj1.disj2.c = Constraint(expr=m.x <= 2)
m.disj1.disjunction = Disjunction(expr=[m.disj1.disj1, m.disj1.disj2])
m.disj2 = Disjunct()
m.disj2.c = Constraint(expr=m.y <= 16)
m.disjunction = Disjunction(expr=[m.disj1, m.disj2])

m.obj = Objective(expr=m.y, sense=maximize)

TransformationFactory('gdp.bigm').apply_to(m, targets=[m.disjunction])
```

### Error Message

```
Traceback (most recent call last):
  File "multipleReferencesToNestedIndVars.py", line 21, in <module>
    TransformationFactory('gdp.bigm').apply_to(m, targets=[m.disjunction])
  File "/home/esjohn/src/pyomo/pyomo/core/base/plugin.py", line 243, in apply_to
    self._apply_to(model, **kwds)
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 208, in _apply_to
    self._apply_to_impl(instance, **kwds)
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 249, in _apply_to_impl
    self._transform_disjunctionData( t, bigM, t.index())
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 392, in _transform_disjunctionData
    suffix_list)
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 463, in _transform_disjunct
    self._transform_block_components(obj, obj, bigM, arg_list, suffix_list)
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 520, in _transform_block_components
    handler(obj, disjunct, bigM, arg_list, suffix_list)
  File "/home/esjohn/src/pyomo/pyomo/gdp/plugins/bigm.py", line 554, in _warn_for_active_disjunct
    _warn_for_active_disjunct(innerdisjunct, outerdisjunct, NAME_BUFFER)
  File "/home/esjohn/src/pyomo/pyomo/gdp/util.py", line 292, in _warn_for_active_disjunct
    name_buffer=NAME_BUFFER)))
pyomo.gdp.disjunct.GDP_Error: Found active disjunct 'disj1.disj1' in disjunct 'disj1'! Either disj1.disj1 is not in a disjunction or the disjunction it is in has not been transformed. disj1.disj1 needs to be deactivated or its disjunction transformed before disj1 can be transformed.
```

### Information on your system

Pyomo version: main branch
Python version: 3.6.5
Operating system: linux
How Pyomo was installed (PyPI, conda, source): source
Solver (if applicable):


### Additional information

<!-- If you have any additional information, please list it here. -->


<!-- We encourage you to try, as much as possible, to reduce your problem to the minimal example that still reproduces the issue. That would help us a lot in fixing it quickly and effectively!

If you want to ask a question (how to use Pyomo, what it can do, etc.), please refer to our [online documentation](https://github.com/Pyomo/pyomo#getting-help) on ways to get help. -->




Discussion:

Resolving patch:
diff --git a/pyomo/gdp/plugins/bigm.py b/pyomo/gdp/plugins/bigm.py
--- a/pyomo/gdp/plugins/bigm.py
+++ b/pyomo/gdp/plugins/bigm.py
@@ -31,9 +31,8 @@
                              get_src_constraint, get_transformed_constraints,
                              _get_constraint_transBlock, get_src_disjunct,
                              _warn_for_active_disjunction,
-                             _warn_for_active_disjunct, )
+                             _warn_for_active_disjunct, preprocess_targets)
 from pyomo.repn import generate_standard_repn
-
 from functools import wraps
 from weakref import ref as weakref_ref
 
@@ -213,6 +212,12 @@ def _apply_to(self, instance, **kwds):
             self.used_args.clear()
 
     def _apply_to_impl(self, instance, **kwds):
+        if not instance.ctype in (Block, Disjunct):
+            raise GDP_Error("Transformation called on %s of type %s. 'instance' "
+                            "must be a ConcreteModel, Block, or Disjunct (in "
+                            "the case of nested disjunctions)." %
+                            (instance.name, instance.ctype))
+
         config = self.CONFIG(kwds.pop('options', {}))
 
         # We will let args override suffixes and estimate as a last
@@ -231,9 +236,15 @@ def _apply_to_impl(self, instance, **kwds):
         targets = config.targets
         if targets is None:
             targets = (instance, )
-        # We need to check that all the targets are in fact on instance. As we
-        # do this, we will use the set below to cache components we know to be
-        # in the tree rooted at instance.
+        else:
+            # we need to preprocess targets to make sure that if there are any
+            # disjunctions in targets that their disjuncts appear before them in
+            # the list.
+            targets = preprocess_targets(targets)
+
+        #  We need to check that all the targets are in fact on
+        # instance. As we do this, we will use the set below to cache components
+        # we know to be in the tree rooted at instance.
         knownBlocks = {}
         for t in targets:
             # check that t is in fact a child of instance
diff --git a/pyomo/gdp/plugins/hull.py b/pyomo/gdp/plugins/hull.py
--- a/pyomo/gdp/plugins/hull.py
+++ b/pyomo/gdp/plugins/hull.py
@@ -29,7 +29,7 @@
                              is_child_of, get_src_disjunction,
                              get_src_constraint, get_transformed_constraints,
                              get_src_disjunct, _warn_for_active_disjunction,
-                             _warn_for_active_disjunct, )
+                             _warn_for_active_disjunct, preprocess_targets)
 from functools import wraps
 from weakref import ref as weakref_ref
 
@@ -230,6 +230,12 @@ def _apply_to(self, instance, **kwds):
             NAME_BUFFER.clear()
 
     def _apply_to_impl(self, instance, **kwds):
+        if not instance.ctype in (Block, Disjunct):
+            raise GDP_Error("Transformation called on %s of type %s. 'instance' "
+                            "must be a ConcreteModel, Block, or Disjunct (in "
+                            "the case of nested disjunctions)." %
+                            (instance.name, instance.ctype))
+
         self._config = self.CONFIG(kwds.pop('options', {}))
         self._config.set_value(kwds)
         self._generate_debug_messages = is_debug_set(logger)
@@ -237,6 +243,11 @@ def _apply_to_impl(self, instance, **kwds):
         targets = self._config.targets
         if targets is None:
             targets = ( instance, )
+        else:
+            # we need to preprocess targets to make sure that if there are any
+            # disjunctions in targets that their disjuncts appear before them in
+            # the list.
+            targets = preprocess_targets(targets)
         knownBlocks = {}
         for t in targets:
             # check that t is in fact a child of instance
diff --git a/pyomo/gdp/util.py b/pyomo/gdp/util.py
--- a/pyomo/gdp/util.py
+++ b/pyomo/gdp/util.py
@@ -80,7 +80,22 @@ def clone_without_expression_components(expr, substitute=None):
                                                 remove_named_expressions=True)
     return visitor.dfs_postorder_stack(expr)
 
-
+def preprocess_targets(targets):
+    preprocessed_targets = []
+    for t in targets:
+        if t.ctype is Disjunction:
+            if t.is_indexed():
+                for disjunction in t.values():
+                    for disj in disjunction.disjuncts:
+                        preprocessed_targets.append(disj)
+            else:
+                for disj in t.disjuncts:
+                    preprocessed_targets.append(disj)
+        # now we are safe to put the disjunction, and if the target was
+        # anything else, then we don't need to worry because disjuncts
+        # are declared before disjunctions they appear in
+        preprocessed_targets.append(t)
+    return preprocessed_targets
 
 def target_list(x):
     if isinstance(x, _ComponentBase):


Test patch:
diff --git a/pyomo/gdp/tests/common_tests.py b/pyomo/gdp/tests/common_tests.py
--- a/pyomo/gdp/tests/common_tests.py
+++ b/pyomo/gdp/tests/common_tests.py
@@ -14,12 +14,13 @@
                            value)
 from pyomo.gdp import Disjunct, Disjunction, GDP_Error
 from pyomo.core.base import constraint, ComponentUID
+from pyomo.core.base.block import _BlockData
 from pyomo.repn import generate_standard_repn
 import pyomo.gdp.tests.models as models
 from io import StringIO
 import random
 
-# utitility functions
+# utility functions
 
 def check_linear_coef(self, repn, var, coef):
     # utility used to check a variable-coefficient pair in a standard_repn
@@ -507,21 +508,6 @@ def check_only_targets_get_transformed(self, transformation):
     self.assertIsNone(m.disjunct2[0].transformation_block)
     self.assertIsNone(m.disjunct2[1].transformation_block)
 
-def check_targets_with_container_as_arg(self, transformation):
-    # check that we can give a Disjunction as the argument to the transformation
-    # and use targets to specify a DisjunctionData to transform
-    m = models.makeTwoTermIndexedDisjunction()
-    TransformationFactory('gdp.%s' % transformation).apply_to(
-        m.disjunction,
-        targets=(m.disjunction[2]))
-    transBlock = m.component("_pyomo_gdp_%s_reformulation" % transformation)
-    self.assertIsNone(m.disjunction[1].algebraic_constraint)
-    self.assertIsNone(m.disjunction[3].algebraic_constraint)
-    self.assertIs(m.disjunction[2].algebraic_constraint(),
-                  transBlock.disjunction_xor[2])
-    self.assertIs(m.disjunction._algebraic_constraint(),
-                  transBlock.disjunction_xor)
-
 def check_target_not_a_component_error(self, transformation):
     # test error message for crazy targets
     decoy = ConcreteModel()
@@ -619,6 +605,9 @@ def innerdisj_rule(d, flag):
     m.disjunct1[1,1].innerdisjunction = Disjunction([0],
         rule=lambda a,i: [m.disjunct1[1,1].innerdisjunct[0],
                           m.disjunct1[1,1].innerdisjunct[1]])
+    # if the disjunction doesn't drive the transformation of the Disjuncts, we
+    # get the error
+    m.disjunct1[1,1].innerdisjunction.deactivate()
     # This test relies on the order that the component objects of
     # the disjunct get considered. In this case, the disjunct
     # causes the error, but in another world, it could be the
@@ -630,30 +619,7 @@ def innerdisj_rule(d, flag):
         TransformationFactory('gdp.%s' % transformation).create_using,
         m,
         targets=[m.disjunction1[1]])
-    #
-    # we will make that disjunction come first now...
-    #
-    tmp = m.disjunct1[1,1].innerdisjunct
-    m.disjunct1[1,1].del_component(tmp)
-    m.disjunct1[1,1].add_component('innerdisjunct', tmp)
-    self.assertRaisesRegex(
-        GDP_Error,
-        r"Found untransformed disjunction 'disjunct1\[1,1\]."
-        r"innerdisjunction\[0\]' in disjunct 'disjunct1\[1,1\]'!.*",
-        TransformationFactory('gdp.%s' % transformation).create_using,
-        m,
-        targets=[m.disjunction1[1]])
-    # Deactivating the disjunction will allow us to get past it back
-    # to the Disjunct (after we realize there are no active
-    # DisjunctionData within the active Disjunction)
-    m.disjunct1[1,1].innerdisjunction[0].deactivate()
-    self.assertRaisesRegex(
-        GDP_Error,
-        r"Found active disjunct 'disjunct1\[1,1\].innerdisjunct\[0\]' "
-        r"in disjunct 'disjunct1\[1,1\]'!.*",
-        TransformationFactory('gdp.%s' % transformation).create_using,
-        m,
-        targets=[m.disjunction1[1]])
+    m.disjunct1[1,1].innerdisjunction.activate()
 
 def check_disjData_targets_inactive(self, transformation):
     # check targets deactivated with DisjunctionData is the target
@@ -1267,6 +1233,20 @@ def check_error_for_same_disjunct_in_multiple_disjunctions(self, transformation)
         TransformationFactory('gdp.%s' % transformation).apply_to,
         m)
 
+def check_cannot_call_transformation_on_disjunction(self, transformation):
+    m = models.makeTwoTermIndexedDisjunction()
+    trans = TransformationFactory('gdp.%s' % transformation)
+    self.assertRaisesRegex(
+        GDP_Error,
+        r"Transformation called on disjunction of type "
+        r"<class 'pyomo.gdp.disjunct.Disjunction'>. 'instance' "
+        r"must be a ConcreteModel, Block, or Disjunct \(in "
+        r"the case of nested disjunctions\).",
+        trans.apply_to,
+        m.disjunction, 
+        targets=m.disjunction[1]
+    )
+
 # This is really neurotic, but test that we will create an infeasible XOR
 # constraint. We have to because in the case of nested disjunctions, our model
 # is not necessarily infeasible because of this. It just might make a Disjunct
@@ -1308,6 +1288,9 @@ def setup_infeasible_xor_because_all_disjuncts_deactivated(self, transformation)
 
 def check_disjunction_target_err(self, transformation):
     m = models.makeNestedDisjunctions()
+    # deactivate the disjunction that would transform the nested Disjuncts so
+    # that we see it is possible to get the error.
+    m.simpledisjunct.innerdisjunction.deactivate()
     self.assertRaisesRegex(
         GDP_Error,
         "Found active disjunct 'simpledisjunct.innerdisjunct0' in "
@@ -1316,20 +1299,6 @@ def check_disjunction_target_err(self, transformation):
         m,
         targets=[m.disjunction])
 
-def check_activeInnerDisjunction_err(self, transformation):
-    m = models.makeDuplicatedNestedDisjunction()
-    self.assertRaisesRegex(
-        GDP_Error,
-        r"Found untransformed disjunction "
-        r"'outerdisjunct\[1\].duplicateddisjunction' in disjunct "
-        r"'outerdisjunct\[1\]'! The disjunction must be transformed before "
-        r"the disjunct. If you are using targets, put the disjunction "
-        r"before the disjunct in the list.*",
-        TransformationFactory('gdp.%s' % transformation).apply_to,
-        m,
-        targets=[m.outerdisjunct[1].innerdisjunction,
-                 m.disjunction])
-
 
 # nested disjunctions: hull and bigm have very different handling for nested
 # disjunctions, but these tests check *that* everything is transformed, not how
@@ -1505,6 +1474,66 @@ def check_disjunctData_only_targets_transformed(self, transformation):
         self.assertIs(m.disjunct[1].innerdisjunct[i].transformation_block(),
                       disjBlock[j])
 
+def check_all_components_transformed(self, m):
+    # checks that all the disjunctive components claim to be transformed in the
+    # makeNestedDisjunctions_NestedDisjuncts model.
+    self.assertIsInstance(m.disj.algebraic_constraint(), Constraint)
+    self.assertIsInstance(m.d1.disj2.algebraic_constraint(), Constraint)
+    self.assertIsInstance(m.d1.transformation_block(), _BlockData)
+    self.assertIsInstance(m.d2.transformation_block(), _BlockData)
+    self.assertIsInstance(m.d1.d3.transformation_block(), _BlockData)
+    self.assertIsInstance(m.d1.d4.transformation_block(), _BlockData)
+
+def check_transformation_blocks_nestedDisjunctions(self, m, transformation):
+    disjunctionTransBlock = m.disj.algebraic_constraint().parent_block()
+    transBlocks = disjunctionTransBlock.relaxedDisjuncts
+    self.assertTrue(len(transBlocks), 4)
+    self.assertIs(transBlocks[0], m.d1.transformation_block())
+    self.assertIs(transBlocks[3], m.d2.transformation_block())
+    if transformation == 'bigm':
+        # we moved the blocks up
+        self.assertIs(transBlocks[1], m.d1.d3.transformation_block())
+        self.assertIs(transBlocks[2], m.d1.d4.transformation_block())
+    if transformation == 'hull':
+        # we only moved the references up, these still point to the inner
+        # transformation blocks
+        inner = m.d1.disj2.algebraic_constraint().parent_block().\
+                relaxedDisjuncts
+        self.assertIs(inner[0], m.d1.d3.transformation_block())
+        self.assertIs(inner[1], m.d1.d4.transformation_block())
+
+def check_nested_disjunction_target(self, transformation):
+    m = models.makeNestedDisjunctions_NestedDisjuncts()
+    transform = TransformationFactory('gdp.%s' % transformation)
+    transform.apply_to(m, targets=[m.disj])
+
+    # the bug that inspired this test throws an error while doing the
+    # transformation, so we'll just do a quick check that all the GDP
+    # components think they are transformed.
+    check_all_components_transformed(self, m)
+    check_transformation_blocks_nestedDisjunctions(self, m, transformation)
+
+def check_target_appears_twice(self, transformation):
+    m = models.makeNestedDisjunctions_NestedDisjuncts()
+    # Because of the way we preprocess targets, the result here will be that
+    # m.d1 appears twice in the list of targets. However, this is fine because
+    # the transformation will not try to retransform anything that has already
+    # been transformed.
+    m1 = TransformationFactory('gdp.%s' % transformation).create_using(
+        m, targets=[m.d1, m.disj])
+    
+    check_all_components_transformed(self, m1)
+    # check we have correct number of transformation blocks
+    check_transformation_blocks_nestedDisjunctions(self, m1, transformation)
+
+    # Now check the same thing, but if the already-transformed disjunct appears
+    # after its disjunction.
+    TransformationFactory('gdp.%s' % transformation).apply_to( m,
+                                                               targets=[m.disj,
+                                                                        m.d1])
+    check_all_components_transformed(self, m)
+    check_transformation_blocks_nestedDisjunctions(self, m, transformation)
+
 def check_unique_reference_to_nested_indicator_var(self, transformation):
     m = models.makeNestedDisjunctions_NestedDisjuncts()
     TransformationFactory('gdp.%s' % transformation).apply_to(m)
diff --git a/pyomo/gdp/tests/models.py b/pyomo/gdp/tests/models.py
--- a/pyomo/gdp/tests/models.py
+++ b/pyomo/gdp/tests/models.py
@@ -541,9 +541,7 @@ def d2_rule(disjunct, flag):
 def makeDuplicatedNestedDisjunction():
     """Not a transformable model (because of disjuncts shared between 
     disjunctions): A SimpleDisjunction where one of the disjuncts contains
-    two SimpleDisjunctions with the same Disjuncts. This is a lazy
-    way to test that we complain about untransformed disjunctions we encounter
-    while transforming a disjunct.
+    two SimpleDisjunctions with the same Disjuncts.
     """
     m = ConcreteModel()
     m.x = Var(bounds=(0, 8))
diff --git a/pyomo/gdp/tests/test_bigm.py b/pyomo/gdp/tests/test_bigm.py
--- a/pyomo/gdp/tests/test_bigm.py
+++ b/pyomo/gdp/tests/test_bigm.py
@@ -702,9 +702,6 @@ def test_create_using(self):
         m = models.makeTwoTermMultiIndexedDisjunction()
         self.diff_apply_to_and_create_using(m)
 
-    def test_targets_with_container_as_arg(self):
-        ct.check_targets_with_container_as_arg(self, 'bigm')
-
 class DisjOnBlock(unittest.TestCase, CommonTests):
     # when the disjunction is on a block, we want all of the stuff created by
     # the transformation to go on that block also so that solving the block
@@ -1974,9 +1971,18 @@ def test_disjunctData_targets_inactive(self):
     def test_disjunctData_only_targets_transformed(self):
         ct.check_disjunctData_only_targets_transformed(self, 'bigm')
 
+    def test_cannot_call_transformation_on_disjunction(self):
+        ct.check_cannot_call_transformation_on_disjunction(self, 'bigm')
+
     def test_disjunction_target_err(self):
         ct.check_disjunction_target_err(self, 'bigm')
 
+    def test_nested_disjunction_target(self):
+        ct.check_nested_disjunction_target(self, 'bigm')
+
+    def test_target_appears_twice(self):
+        ct.check_target_appears_twice(self, 'bigm')
+
     def test_create_using(self):
         m = models.makeNestedDisjunctions()
         self.diff_apply_to_and_create_using(m)
@@ -2160,10 +2166,6 @@ def test_use_correct_none_suffix(self):
         self.assertIs(repn.linear_vars[1], m.b.d.indicator_var)
         self.assertEqual(repn.linear_coefs[1], -10)
 
-class InnerDisjunctionSharedDisjuncts(unittest.TestCase):
-    def test_activeInnerDisjunction_err(self):
-        ct.check_activeInnerDisjunction_err(self, 'bigm')
-
 class UntransformableObjectsOnDisjunct(unittest.TestCase):
     def test_RangeSet(self):
         ct.check_RangeSet(self, 'bigm')
diff --git a/pyomo/gdp/tests/test_hull.py b/pyomo/gdp/tests/test_hull.py
--- a/pyomo/gdp/tests/test_hull.py
+++ b/pyomo/gdp/tests/test_hull.py
@@ -751,8 +751,8 @@ def test_disjunction_data_target(self):
     def test_disjunction_data_target_any_index(self):
         ct.check_disjunction_data_target_any_index(self, 'hull')
 
-    def test_targets_with_container_as_arg(self):
-        ct.check_targets_with_container_as_arg(self, 'hull')
+    def test_cannot_call_transformation_on_disjunction(self):
+        ct.check_cannot_call_transformation_on_disjunction(self, 'hull')
 
     def check_trans_block_disjunctions_of_disjunct_datas(self, m):
         transBlock1 = m.component("_pyomo_gdp_hull_reformulation")
@@ -1078,6 +1078,12 @@ def test_disjunctData_only_targets_transformed(self):
     def test_disjunction_target_err(self):
         ct.check_disjunction_target_err(self, 'hull')
 
+    def test_nested_disjunction_target(self):
+        ct.check_nested_disjunction_target(self, 'hull')
+
+    def test_target_appears_twice(self):
+        ct.check_target_appears_twice(self, 'hull')
+
     @unittest.skipIf(not linear_solvers, "No linear solver available")
     def test_relaxation_feasibility(self):
         m = models.makeNestedDisjunctions_FlatDisjuncts()
@@ -1538,7 +1544,7 @@ def test_solve_nested_model(self):
 
         # transform inner problem with bigm, outer with hull and make sure it
         # still works
-        TransformationFactory('gdp.bigm').apply_to(m.d1.disj2)
+        TransformationFactory('gdp.bigm').apply_to(m, targets=(m.d1.disj2))
         hull.apply_to(m)
 
         SolverFactory(linear_solvers[0]).solve(m)
@@ -1915,10 +1921,6 @@ def test_mapping_method_errors(self):
             m.w,
             m.random_disjunction.disjuncts[0])
 
-class InnerDisjunctionSharedDisjuncts(unittest.TestCase):
-    def test_activeInnerDisjunction_err(self):
-        ct.check_activeInnerDisjunction_err(self, 'hull')
-
 class BlocksOnDisjuncts(unittest.TestCase):
     def setUp(self):
         # set seed so we can test name collisions predictably