Repository: Pyomo/pyomo

Issue:
Cloning models with references does not preserve the references correctly
The ```_data``` attribute of a reference is of type ```_ReferenceDict``` which has a ```_slice``` attribute which should be an ```_IndexedComponent_slice``` object. However, when a model gets cloned, this ```_slice``` attribute becomes a list. This causes an error when iterating over components in the model:

```
>>> import pyomo.environ as pe
>>> m = pe.ConcreteModel()
>>> m.x = pe.Var()
>>> m.x_ref = pe.Reference(m.x)
>>> m2 = m.clone()
>>> m.x_ref._data._slice
<pyomo.core.base.indexed_component_slice._IndexedComponent_slice object at 0x151fa63438>
>>> m2.x_ref._data._slice
[<pyomo.core.base.var.SimpleVar object at 0x152002b048>]
>>> for v in m.component_data_objects(pe.Var):
...     print(v)
... 
x
x
>>> for v in m2.component_data_objects(pe.Var):
...     print(v)
... 
x
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/.../pyomo/core/base/block.py", line 1342, in component_data_objects
    sort=sort):
  File "/Users/.../pyomo/core/base/block.py", line 1274, in _component_data_iter
    for idx, compData in _items:
  File "/Users/.../pyomo/core/base/indexed_component.py", line 309, in iteritems
    yield key, self[key]
  File "/Users/.../pyomo/core/base/indexed_component.py", line 319, in __getitem__
    obj = self._data.get(index, _NotFound)
  File "/Users/.../_collections_abc.py", line 660, in get
    return self[key]
  File "/Users/.../pyomo/core/base/reference.py", line 178, in __getitem__
    self._get_iter(self._slice, key, get_if_not_present=True)
  File "/Users/.../pyomo/core/base/reference.py", line 292, in _get_iter
    get_if_not_present=get_if_not_present)
  File "/Users/.../pyomo/core/base/indexed_component_slice.py", line 265, in __init__
    call_stack = self._slice._call_stack
AttributeError: 'list' object has no attribute '_call_stack'

```
Cloning models with references does not preserve the references correctly
The ```_data``` attribute of a reference is of type ```_ReferenceDict``` which has a ```_slice``` attribute which should be an ```_IndexedComponent_slice``` object. However, when a model gets cloned, this ```_slice``` attribute becomes a list. This causes an error when iterating over components in the model:

```
>>> import pyomo.environ as pe
>>> m = pe.ConcreteModel()
>>> m.x = pe.Var()
>>> m.x_ref = pe.Reference(m.x)
>>> m2 = m.clone()
>>> m.x_ref._data._slice
<pyomo.core.base.indexed_component_slice._IndexedComponent_slice object at 0x151fa63438>
>>> m2.x_ref._data._slice
[<pyomo.core.base.var.SimpleVar object at 0x152002b048>]
>>> for v in m.component_data_objects(pe.Var):
...     print(v)
... 
x
x
>>> for v in m2.component_data_objects(pe.Var):
...     print(v)
... 
x
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/.../pyomo/core/base/block.py", line 1342, in component_data_objects
    sort=sort):
  File "/Users/.../pyomo/core/base/block.py", line 1274, in _component_data_iter
    for idx, compData in _items:
  File "/Users/.../pyomo/core/base/indexed_component.py", line 309, in iteritems
    yield key, self[key]
  File "/Users/.../pyomo/core/base/indexed_component.py", line 319, in __getitem__
    obj = self._data.get(index, _NotFound)
  File "/Users/.../_collections_abc.py", line 660, in get
    return self[key]
  File "/Users/.../pyomo/core/base/reference.py", line 178, in __getitem__
    self._get_iter(self._slice, key, get_if_not_present=True)
  File "/Users/.../pyomo/core/base/reference.py", line 292, in _get_iter
    get_if_not_present=get_if_not_present)
  File "/Users/.../pyomo/core/base/indexed_component_slice.py", line 265, in __init__
    call_stack = self._slice._call_stack
AttributeError: 'list' object has no attribute '_call_stack'

```


Discussion:
Ah, I can see some reasons why this might be problematic.
The problem appears to be that ```_IndexedComponent_slice.__getattr__``` returns ```self``` even for attributes such as ```__deepcopy__``` and ```__getstate__```.
Yeah - this is definitely a bug.  A patch / PR should be easy to put together - I hope it will be up in an hour or two.
@jsiirola Thanks. I have been trying to fix it myself, but I have been unsuccessful so far.
Ah, I can see some reasons why this might be problematic.
The problem appears to be that ```_IndexedComponent_slice.__getattr__``` returns ```self``` even for attributes such as ```__deepcopy__``` and ```__getstate__```.
Yeah - this is definitely a bug.  A patch / PR should be easy to put together - I hope it will be up in an hour or two.
@jsiirola Thanks. I have been trying to fix it myself, but I have been unsuccessful so far.

Resolving patch:
diff --git a/pyomo/core/base/component.py b/pyomo/core/base/component.py
--- a/pyomo/core/base/component.py
+++ b/pyomo/core/base/component.py
@@ -202,10 +202,14 @@ def __deepcopy__(self, memo):
                             'paranoid' mode by adding '__paranoid__' ==
                             True to the memo before calling
                             copy.deepcopy.""")
+                    if self.model() is self:
+                        what = 'Model'
+                    else:
+                        what = 'Component'
                     logger.error(
                         "Unable to clone Pyomo component attribute.\n"
-                        "Component '%s' contains an uncopyable field '%s' (%s)"
-                        % ( self.name, k, type(v) ))
+                        "%s '%s' contains an uncopyable field '%s' (%s)"
+                        % ( what, self.name, k, type(v) ))
         ans.__setstate__(new_state)
         return ans
 
diff --git a/pyomo/core/base/indexed_component_slice.py b/pyomo/core/base/indexed_component_slice.py
--- a/pyomo/core/base/indexed_component_slice.py
+++ b/pyomo/core/base/indexed_component_slice.py
@@ -7,7 +7,7 @@
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
-
+import copy
 from six import PY3, iteritems, advance_iterator
 from pyomo.common import DeveloperError
 
@@ -47,6 +47,29 @@ def __init__(self, component, fixed, sliced, ellipsis):
         set_attr('key_errors_generate_exceptions', True)
         set_attr('attribute_errors_generate_exceptions', True)
 
+    def __getstate__(self):
+        """Serialize this object.
+
+        In general, we would not need to implement this (the object does
+        not leverage ``__slots__``).  However, because we have a
+        "blanket" implementation of :py:meth:`__getattr__`, we need to
+        explicitly implement these to avoid "accidentally" extending or
+        evaluating this slice."""
+        return dict(
+            (k,getattr(self,k)) for k in self.__dict__)
+
+    def __setstate__(self, state):
+        """Deserialize the state into this object. """
+        set_attr = super(_IndexedComponent_slice, self).__setattr__
+        for k,v in iteritems(state):
+            set_attr(k,v)
+
+    def __deepcopy__(self, memo):
+        """Deepcopy this object (leveraging :py:meth:`__getstate__`)"""
+        ans = memo[id(self)] = self.__class__.__new__(self.__class__)
+        ans.__setstate__(copy.deepcopy(self.__getstate__(), memo))
+        return ans
+
     def __iter__(self):
         """Return an iterator over this slice"""
         return _IndexedComponent_slice_iter(self)


Test patch:
diff --git a/pyomo/core/tests/unit/test_indexed_slice.py b/pyomo/core/tests/unit/test_indexed_slice.py
--- a/pyomo/core/tests/unit/test_indexed_slice.py
+++ b/pyomo/core/tests/unit/test_indexed_slice.py
@@ -12,35 +12,43 @@
 #
 
 import os
-from os.path import abspath, dirname
-currdir = dirname(abspath(__file__))+os.sep
+import pickle
 
 import pyutilib.th as unittest
-from six import itervalues
 
 from pyomo.environ import *
 from pyomo.core.base.block import _BlockData
 from pyomo.core.base.indexed_component import _IndexedComponent_slice
 
+def _x_init(m, k):
+    return k
 
-class TestComponentSlices(unittest.TestCase):
-    def setUp(self):
-        def _c(b, i, j):
-            b.x = Var(b.model().K, initialize=lambda m,k: i*100+j*10+k)
+def _y_init(m, i, j):
+    return i*10+j
+
+def _cx_init(b, k):
+    i, j = b.index()[:2]
+    return i*100+j*10+k
+
+def _c(b, i, j):
+    b.x = Var(b.model().K, initialize=_cx_init)
+
+def _b(b, i, j):
+    _c(b,i,j)
+    b.c = Block(b.model().I, b.model().J, rule=_c)
 
-        def _b(b, i, j):
-            _c(b,i,j)
-            b.c = Block(b.model().I, b.model().J, rule=_c)
-        def _bb(b, i, j, k):
-            _c(b,i,j)
-            b.c = Block(b.model().I, b.model().J, rule=_c)
+def _bb(b, i, j, k):
+    _c(b,i,j)
+    b.c = Block(b.model().I, b.model().J, rule=_c)
 
+class TestComponentSlices(unittest.TestCase):
+    def setUp(self):
         self.m = m = ConcreteModel()
         m.I = RangeSet(1,3)
         m.J = RangeSet(4,6)
         m.K = RangeSet(7,9)
-        m.x = Var(m.K, initialize=lambda m,k: k)
-        m.y = Var(m.I, m.J, initialize=lambda m,i,j: i*10+j)
+        m.x = Var(m.K, initialize=_x_init)
+        m.y = Var(m.I, m.J, initialize=_y_init)
         m.b = Block(m.I, m.J, rule=_b)
         m.bb = Block(m.I, m.J, m.K, rule=_bb)
 
@@ -461,7 +469,50 @@ def test_iterators(self):
             [((1,4), m.b[1,4]), ((1,5), m.b[1,5]), ((1,6), m.b[1,6])]
         )
 
-
+    def test_pickle_slices(self):
+        m = self.m
+        _slicer = m.b[1,:].c[:,4].x
+        _new_slicer = pickle.loads(pickle.dumps(_slicer))
+
+        self.assertIsNot(_slicer, _new_slicer)
+        self.assertIsNot(_slicer._call_stack, _new_slicer._call_stack)
+        self.assertIs(type(_slicer._call_stack), type(_new_slicer._call_stack))
+        self.assertEqual(len(_slicer._call_stack), len(_new_slicer._call_stack))
+
+        ref = ['b[1,4].c[1,4].x', 'b[1,4].c[2,4].x', 'b[1,4].c[3,4].x',
+               'b[1,5].c[1,4].x', 'b[1,5].c[2,4].x', 'b[1,5].c[3,4].x',
+               'b[1,6].c[1,4].x', 'b[1,6].c[2,4].x', 'b[1,6].c[3,4].x',
+               ]
+        self.assertEqual([str(x) for x in _slicer], ref )
+        self.assertEqual([str(x) for x in _new_slicer], ref )
+        for x,y in zip(iter(_slicer), iter(_new_slicer)):
+            self.assertIs(type(x), type(y))
+            self.assertEqual(x.name, y.name)
+            self.assertIsNot(x, y)
+
+    def test_clone_on_model(self):
+        m = self.m
+        m.slicer = m.b[1,:].c[:,4].x
+        n = m.clone()
+
+        self.assertIsNot(m, n)
+        self.assertIsNot(m.slicer, n.slicer)
+        self.assertIsNot(m.slicer._call_stack, n.slicer._call_stack)
+        self.assertIs(type(m.slicer._call_stack), type(n.slicer._call_stack))
+        self.assertEqual(len(m.slicer._call_stack), len(n.slicer._call_stack))
+
+        ref = ['b[1,4].c[1,4].x', 'b[1,4].c[2,4].x', 'b[1,4].c[3,4].x',
+               'b[1,5].c[1,4].x', 'b[1,5].c[2,4].x', 'b[1,5].c[3,4].x',
+               'b[1,6].c[1,4].x', 'b[1,6].c[2,4].x', 'b[1,6].c[3,4].x',
+               ]
+        self.assertEqual([str(x) for x in m.slicer], ref )
+        self.assertEqual([str(x) for x in n.slicer], ref )
+        for x,y in zip(iter(m.slicer), iter(n.slicer)):
+            self.assertIs(type(x), type(y))
+            self.assertEqual(x.name, y.name)
+            self.assertIsNot(x, y)
+            self.assertIs(x.model(), m)
+            self.assertIs(y.model(), n)
 
 if __name__ == "__main__":
     unittest.main()