Repository: keras-team/keras

Issue:
Keras uses an exponential time algorithm for topological sorting.
- [x] Up-to-date with the master branch of Keras.
- The backend is not involved.

Example code illustrating the issue:
```python
from keras.layers import Dense, Input, add
from keras.models import Model

def make_left_tower(n):
	x = input = Input((1,))
	dense = Dense(1)
	for _ in range(n):
		x = add([dense(x), x])
	return Model(inputs=[input], outputs=[x])

def make_right_tower(n):
	x = input = Input((1,))
	dense = Dense(1)
	for _ in range(n):
		x = add([x, dense(x)])
	return Model(inputs=[input], outputs=[x])
```

`make_left_tower(100)` runs in under a second on my machine, while `make_right_tower(16)` already takes longer, and doubles its runtime as depth increases.

The problem is in the [`build_map_of_graph` function](https://github.com/fchollet/keras/blob/18ed60b9f29fc061aa66d53e820c38a12c4f6688/keras/engine/topology.py#L1608), which seems intended to compute a [topological sort using the depth-first algorithm](https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search), but uses a non-standard marking scheme for the visited nodes. This leads to the observed pathological behavior where subgraphs are traversed multiple times if they are seen at successively higher depths (as is the case for the models created by `make_right_tower`).

I will work on a PR to fix this issue, but beforehand I'd like to know how important it is that the computed order be identical to the current result (the [comment about determinism](https://github.com/fchollet/keras/blob/18ed60b9f29fc061aa66d53e820c38a12c4f6688/keras/engine/topology.py#L1693) makes we worry someone's weights will be swapped when loading a model generated with a different ordering, or something.)


Discussion:

Resolving patch:
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1608,56 +1608,53 @@ def __init__(self, inputs, outputs, name=None):
         nodes_depths = {}  # dict {node: depth value}
         layers_depths = {}  # dict {layer: depth value}
         layer_indices = {}  # dict {layer: index in traversal}
+        nodes_in_decreasing_depth = []
 
-        def make_node_marker(node, depth):
-            return str(id(node)) + '-' + str(depth)
-
-        def build_map_of_graph(tensor, seen_nodes=None, depth=0,
+        def build_map_of_graph(tensor, finished_nodes, nodes_in_progress,
                                layer=None, node_index=None, tensor_index=None):
             """Builds a map of the graph of layers.
 
-            This recursively updates the maps `nodes_depths`,
-            `layers_depths` and the set `container_nodes`.
-
-            Does not try to detect cycles in the graph.
+            This recursively updates the map `layer_indices`,
+            the list `nodes_in_decreasing_depth` and the set `container_nodes`.
 
             # Arguments
                 tensor: Some tensor in a graph.
-                seen_nodes: Set of node ids ("{layer.name}_ib-{node_index}")
-                    of nodes seen so far. Useful to prevent infinite loops.
-                depth: Current depth in the graph (0 = last output).
+                finished_nodes: Set of nodes whose subgraphs have been traversed
+                    completely. Useful to prevent duplicated work.
+                nodes_in_progress: Set of nodes that are currently active on the
+                    recursion stack. Useful to detect cycles.
                 layer: Layer from which `tensor` comes from. If not provided,
                     will be obtained from `tensor._keras_history`.
                 node_index: Node index from which `tensor` comes from.
                 tensor_index: Tensor_index from which `tensor` comes from.
+
+            # Raises
+                RuntimeError: if a cycle is detected.
             """
-            seen_nodes = seen_nodes or set()
             if not layer or node_index is None or tensor_index is None:
                 layer, node_index, tensor_index = tensor._keras_history
             node = layer.inbound_nodes[node_index]
 
             # Prevent cycles.
-            seen_nodes.add(make_node_marker(node, depth))
+            if node in nodes_in_progress:
+                raise RuntimeError(
+                    'The tensor ' + str(tensor) + ' at layer "' +
+                    layer.name + '" is part of a cycle.')
+
+            # Don't repeat work for shared subgraphs
+            if node in finished_nodes:
+                return
 
             node_key = layer.name + '_ib-' + str(node_index)
             # Update container_nodes.
             container_nodes.add(node_key)
-            # Update nodes_depths.
-            node_depth = nodes_depths.get(node)
-            if node_depth is None:
-                nodes_depths[node] = depth
-            else:
-                nodes_depths[node] = max(depth, node_depth)
-            # Update layers_depths.
-            previously_seen_depth = layers_depths.get(layer)
-            if previously_seen_depth is None:
-                current_depth = depth
-            else:
-                current_depth = max(depth, previously_seen_depth)
-            layers_depths[layer] = current_depth
+
+            # Store the traversal order for layer sorting.
             if layer not in layer_indices:
                 layer_indices[layer] = len(layer_indices)
 
+            nodes_in_progress.add(node)
+
             # Propagate to all previous tensors connected to this node.
             for i in range(len(node.inbound_layers)):
                 x = node.input_tensors[i]
@@ -1665,15 +1662,34 @@ def build_map_of_graph(tensor, seen_nodes=None, depth=0,
                 node_index = node.node_indices[i]
                 tensor_index = node.tensor_indices[i]
                 next_node = layer.inbound_nodes[node_index]
-                # use node_marker to prevent cycles
-                node_marker = make_node_marker(next_node, current_depth + 1)
-                if node_marker not in seen_nodes:
-                    build_map_of_graph(x, seen_nodes, current_depth + 1,
-                                       layer, node_index, tensor_index)
+                build_map_of_graph(x, finished_nodes, nodes_in_progress,
+                                   layer, node_index, tensor_index)
+
+            finished_nodes.add(node)
+            nodes_in_progress.remove(node)
 
+            nodes_in_decreasing_depth.append(node)
+
+        finished_nodes = set()
+        nodes_in_progress = set()
         for x in self.outputs:
-            seen_nodes = set()
-            build_map_of_graph(x, seen_nodes, depth=0)
+            build_map_of_graph(x, finished_nodes, nodes_in_progress)
+
+        for node in reversed(nodes_in_decreasing_depth):
+            # If the depth is not set, the node has no outbound nodes (depth 0).
+            depth = nodes_depths.setdefault(node, 0)
+
+            # Update the depth of inbound nodes.
+            for i in range(len(node.inbound_layers)):
+                inbound_layer = node.inbound_layers[i]
+                node_index = node.node_indices[i]
+                inbound_node = inbound_layer.inbound_nodes[node_index]
+                previous_depth = nodes_depths.get(inbound_node, 0)
+                nodes_depths[inbound_node] = max(depth + 1, previous_depth)
+
+            # Update the depth of the corresponding layer
+            previous_depth = layers_depths.get(node.outbound_layer, 0)
+            layers_depths[node.outbound_layer] = max(depth, previous_depth)
 
         # Build a dict {depth: list of nodes with this depth}
         nodes_by_depth = {}


Test patch: