Repository: Pyomo/pyomo

Issue:
PySP errors with networkx 2.4
I recently rebuilt the Travis docker images to include an additional package (pathos, see #1122).  As part of that rebuild, some of the docker images picked up a newer version of `networkx` (2.4 instead of 2.3), which appears to have triggered a slew of PySP test failures, e.g.:
```
======================================================================
ERROR: test_bundles (pyomo.pysp.tests.unit.test_scenariotree.TestScenarioTreeFromNetworkX)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/tests/unit/test_scenariotree.py", line 745, in test_bundles
    edge_probability_attribute=None)
  File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 313, in ScenarioTreeModelFromNetworkX
    networkx.dfs_successors(tree, root))
  File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 296, in _setup
    _setup(v, succ)
  File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 311, in _setup
    tree.node[u].get('bundle', None)
AttributeError: 'DiGraph' object has no attribute 'node'
```

For a full build, see: https://travis-ci.org/Pyomo/pyomo/jobs/597959028

@ghackebeil: is this a relatively easy fix, or should we rebuild the Docker images and pin to a `networkx` version before 2.4?


Discussion:
I can’t imagine it’s too difficult of a fix. I’ll take a look.

> On Oct 18, 2019, at 12:37 AM, John Siirola <notifications@github.com> wrote:
> 
> ﻿
> I recently rebuilt the Travis docker images to include an additional package (pathos, see #1122). As part of that rebuild, some of the docker images picked up a newer version of networkx (2.4 instead of 2.3), which appears to have triggered a slew of PySP test failures, e.g.:
> 
> ======================================================================
> ERROR: test_bundles (pyomo.pysp.tests.unit.test_scenariotree.TestScenarioTreeFromNetworkX)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
>   File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/tests/unit/test_scenariotree.py", line 745, in test_bundles
>     edge_probability_attribute=None)
>   File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 313, in ScenarioTreeModelFromNetworkX
>     networkx.dfs_successors(tree, root))
>   File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 296, in _setup
>     _setup(v, succ)
>   File "/home/travis/build/Pyomo/pyomo/pyomo/pysp/scenariotree/tree_structure_model.py", line 311, in _setup
>     tree.node[u].get('bundle', None)
> AttributeError: 'DiGraph' object has no attribute 'node'
> For a full build, see: https://travis-ci.org/Pyomo/pyomo/jobs/597959028
> 
> @ghackebeil: is this a relatively easy fix, or should we rebuild the Docker images and pin to a networkx version before 2.4?
> 
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub, or unsubscribe.


Resolving patch:
diff --git a/pyomo/core/base/indexed_component_slice.py b/pyomo/core/base/indexed_component_slice.py
--- a/pyomo/core/base/indexed_component_slice.py
+++ b/pyomo/core/base/indexed_component_slice.py
@@ -161,6 +161,14 @@ def __call__(self, *idx, **kwds):
         the slice and call the item.  This allows "vector-like" operations
         like: `m.x[:,1].fix(0)`.
         """
+        # There is a weird case in pypy3.6-7.2.0 where __name__ gets
+        # called after retrieving an attribute that will be called.  I
+        # don't know why that happens, but we will trap it here and
+        # remove the getattr(__name__) from the call stack.
+        if self._call_stack[-1][0] == _IndexedComponent_slice.get_attribute \
+           and self._call_stack[-1][1] == '__name__':
+            self._call_stack.pop()
+
         self._call_stack.append( (
             _IndexedComponent_slice.call, idx, kwds ) )
         if self._call_stack[-2][1] == 'component':
diff --git a/pyomo/core/base/util.py b/pyomo/core/base/util.py
--- a/pyomo/core/base/util.py
+++ b/pyomo/core/base/util.py
@@ -54,7 +54,7 @@ def impl(self, *args, **kwds):
     # Python 3.4.  For backwards compatability with Python 2.x, we will
     # create a temporary (lambda) function using eval that matches the
     # function signature passed in and calls the generic impl() function
-    args = inspect.formatargspec(*inspect.getargspec(fcn))
+    args = inspect.formatargspec(*getargspec(fcn))
     impl_args = eval('lambda %s: impl%s' % (args[1:-1], args), {'impl': impl})
     return functools.wraps(fcn)(impl_args)
 
diff --git a/pyomo/pysp/scenariotree/tree_structure_model.py b/pyomo/pysp/scenariotree/tree_structure_model.py
--- a/pyomo/pysp/scenariotree/tree_structure_model.py
+++ b/pyomo/pysp/scenariotree/tree_structure_model.py
@@ -16,7 +16,8 @@
 
 try:
     import networkx
-    has_networkx = True
+    # The code below conforms to the networkx>=2.0 API
+    has_networkx = int(networkx.__version__.split('.')[0]) >= 2
 except ImportError:                               #pragma:nocover
     has_networkx = False
 
@@ -232,7 +233,7 @@ def ScenarioTreeModelFromNetworkX(
 
     if not has_networkx:                          #pragma:nocover
         raise ValueError(
-            "networkx module is not available")
+            "networkx>=2.0 module is not available")
 
     if not networkx.is_tree(tree):
         raise TypeError(
@@ -281,12 +282,12 @@ def ScenarioTreeModelFromNetworkX(
     scenario_bundle = {}
     def _setup(u, succ):
         if node_name_attribute is not None:
-            if node_name_attribute not in tree.node[u]:
+            if node_name_attribute not in tree.nodes[u]:
                 raise KeyError(
                     "node '%s' missing node name "
                     "attribute: '%s'"
                     % (u, node_name_attribute))
-            node_name = tree.node[u][node_name_attribute]
+            node_name = tree.nodes[u][node_name_attribute]
         else:
             node_name = u
         node_to_name[u] = node_name
@@ -297,18 +298,18 @@ def _setup(u, succ):
         else:
             # a leaf node
             if scenario_name_attribute is not None:
-                if scenario_name_attribute not in tree.node[u]:
+                if scenario_name_attribute not in tree.nodes[u]:
                     raise KeyError(
                         "node '%s' missing scenario name "
                         "attribute: '%s'"
                         % (u, scenario_name_attribute))
-                scenario_name = tree.node[u][scenario_name_attribute]
+                scenario_name = tree.nodes[u][scenario_name_attribute]
             else:
                 scenario_name = u
             node_to_scenario[u] = scenario_name
             m.Scenarios.add(scenario_name)
             scenario_bundle[scenario_name] = \
-                tree.node[u].get('bundle', None)
+                tree.nodes[u].get('bundle', None)
     _setup(root,
            networkx.dfs_successors(tree, root))
     m = m.create_instance()
@@ -336,19 +337,19 @@ def _add_node(u, stage, succ, pred):
                 probability = 1.0/len(succ[pred[u]])
             m.ConditionalProbability[node_name] = probability
         # get node variables
-        if "variables" in tree.node[u]:
-            node_variables = tree.node[u]["variables"]
+        if "variables" in tree.nodes[u]:
+            node_variables = tree.nodes[u]["variables"]
             assert type(node_variables) in [tuple, list]
             for varstring in node_variables:
                 m.NodeVariables[node_name].add(varstring)
-        if "derived_variables" in tree.node[u]:
-            node_derived_variables = tree.node[u]["derived_variables"]
+        if "derived_variables" in tree.nodes[u]:
+            node_derived_variables = tree.nodes[u]["derived_variables"]
             assert type(node_derived_variables) in [tuple, list]
             for varstring in node_derived_variables:
                 m.NodeDerivedVariables[node_name].add(varstring)
-        if "cost" in tree.node[u]:
-            assert isinstance(tree.node[u]["cost"], six.string_types)
-            m.NodeCost[node_name].value = tree.node[u]["cost"]
+        if "cost" in tree.nodes[u]:
+            assert isinstance(tree.nodes[u]["cost"], six.string_types)
+            m.NodeCost[node_name].value = tree.nodes[u]["cost"]
         if u in succ:
             child_names = []
             for v in succ[u]:


Test patch:
diff --git a/pyomo/pysp/tests/unit/test_instancefactory.py b/pyomo/pysp/tests/unit/test_instancefactory.py
--- a/pyomo/pysp/tests/unit/test_instancefactory.py
+++ b/pyomo/pysp/tests/unit/test_instancefactory.py
@@ -568,21 +568,21 @@ def test_init17(self):
             self.assertEqual(scenario_tree.contains_bundles(), False)
             # check that we can modify the networkx tree to redefine
             # bundles
-            nx_tree.node["s1"]["bundle"] = 0
-            nx_tree.node["s2"]["bundle"] = 0
-            nx_tree.node["s3"]["bundle"] = 0
+            nx_tree.nodes["s1"]["bundle"] = 0
+            nx_tree.nodes["s2"]["bundle"] = 0
+            nx_tree.nodes["s3"]["bundle"] = 0
             scenario_tree = factory.generate_scenario_tree()
             self.assertEqual(scenario_tree.contains_bundles(), True)
             self.assertEqual(len(scenario_tree.bundles), 1)
-            nx_tree.node["s1"]["bundle"] = 0
-            nx_tree.node["s2"]["bundle"] = 1
-            nx_tree.node["s3"]["bundle"] = 2
+            nx_tree.nodes["s1"]["bundle"] = 0
+            nx_tree.nodes["s2"]["bundle"] = 1
+            nx_tree.nodes["s3"]["bundle"] = 2
             scenario_tree = factory.generate_scenario_tree()
             self.assertEqual(scenario_tree.contains_bundles(), True)
             self.assertEqual(len(scenario_tree.bundles), 3)
-            nx_tree.node["s1"]["bundle"] = None
-            nx_tree.node["s2"]["bundle"] = None
-            nx_tree.node["s3"]["bundle"] = None
+            nx_tree.nodes["s1"]["bundle"] = None
+            nx_tree.nodes["s2"]["bundle"] = None
+            nx_tree.nodes["s3"]["bundle"] = None
             scenario_tree = factory.generate_scenario_tree()
             self.assertEqual(scenario_tree.contains_bundles(), False)
 
diff --git a/pyomo/pysp/tests/unit/test_scenariotree.py b/pyomo/pysp/tests/unit/test_scenariotree.py
--- a/pyomo/pysp/tests/unit/test_scenariotree.py
+++ b/pyomo/pysp/tests/unit/test_scenariotree.py
@@ -787,12 +787,12 @@ def test_bundles_incomplete(self):
         self.assertEqual(list(model.Bundles), ["B"])
         self.assertEqual(list(model.BundleScenarios["B"]),
                          ["u"+str(i) for i in range(4)])
-        G.node["u0"]["bundle"] = None
+        G.nodes["u0"]["bundle"] = None
         with self.assertRaises(ValueError):
             ScenarioTreeModelFromNetworkX(
                 G,
                 edge_probability_attribute=None)
-        del G.node["u0"]["bundle"]
+        del G.nodes["u0"]["bundle"]
         with self.assertRaises(ValueError):
             ScenarioTreeModelFromNetworkX(
                 G,