Repository: conan-io/conan

Issue:
Document better graph_builder code
The graph code is far from being trivial. We have to increase the documentation, maybe each relevant portion of code, like the introduced by #4737 should go with a comment block, even "graphic" like:

```
This is because when A-->B and B--->C bla bla bla
```



Discussion:

Resolving patch:
diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py
--- a/conans/client/graph/graph.py
+++ b/conans/client/graph/graph.py
@@ -36,7 +36,7 @@ def __init__(self, ref, conanfile, recipe=None):
         self.private = False
         self.revision_pinned = False  # The revision has been specified by the user
 
-        # The dependencies that can conflict to downstream consumers
+        # A subset of the graph that will conflict by package name
         self.public_deps = None  # {ref.name: Node}
         # all the public deps only in the closure of this node
         # The dependencies that will be part of deps_cpp_info, can't conflict
@@ -92,6 +92,14 @@ def make_public(self):
             if not edge.private:
                 edge.dst.make_public()
 
+    def connect_closure(self, other_node):
+        # When 2 nodes of the graph become connected, their closures information has
+        # has to remain consistent. This method manages this.
+        name = other_node.name
+        self.public_closure[name] = other_node
+        self.public_deps[name] = other_node
+        other_node.inverse_closure.add(self)
+
     def inverse_neighbors(self):
         return [edge.src for edge in self.dependants]
 
@@ -272,21 +280,18 @@ def _order_levels(self, direct):
         first level nodes, and so on
         return [[node1, node34], [node3], [node23, node8],...]
         """
-        current_level = []
-        result = [current_level]
-        opened = self.nodes.copy()
+        result = []
+        opened = self.nodes
         while opened:
-            current = opened.copy()
+            current_level = []
             for o in opened:
                 o_neighs = o.neighbors() if direct else o.inverse_neighbors()
                 if not any(n in opened for n in o_neighs):
                     current_level.append(o)
-                    current.discard(o)
+
             current_level.sort()
+            result.append(current_level)
             # now initialize new level
-            opened = current
-            if opened:
-                current_level = []
-                result.append(current_level)
+            opened = opened.difference(current_level)
 
         return result
diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py
--- a/conans/client/graph/graph_builder.py
+++ b/conans/client/graph/graph_builder.py
@@ -10,9 +10,6 @@
 from conans.util.log import logger
 
 
-REFERENCE_CONFLICT, REVISION_CONFLICT = 1, 2
-
-
 class DepsGraphBuilder(object):
     """ Responsible for computing the dependencies graph DepsGraph
     """
@@ -63,7 +60,7 @@ def extend_build_requires(self, graph, node, build_requires_refs, check_updates,
             self._handle_require(name, node, require, graph, check_updates, update,
                                  remotes, processed_profile, new_reqs, new_options)
 
-        new_nodes = set([n for n in graph.nodes if n.package_id is None])
+        new_nodes = set(n for n in graph.nodes if n.package_id is None)
         # This is to make sure that build_requires have precedence over the normal requires
         ordered_closure = list(node.public_closure.items())
         ordered_closure.sort(key=lambda x: x[1] not in new_nodes)
@@ -78,12 +75,10 @@ def extend_build_requires(self, graph, node, build_requires_refs, check_updates,
 
         return subgraph
 
-    def _resolve_ranges(self, graph, requires, scope, update, remotes):
-        for require in requires:
-            self._resolver.resolve(require, scope, update, remotes)
-
-        # After resolving ranges,
+    def _resolve_ranges(self, graph, requires, consumer, update, remotes):
         for require in requires:
+            self._resolver.resolve(require, consumer, update, remotes)
+            # if the range is resolved, check if it is an alias
             alias = graph.aliased.get(require.ref)
             if alias:
                 require.ref = alias
@@ -130,103 +125,103 @@ def _load_deps(self, dep_graph, node, down_reqs, down_ref, down_options,
 
     def _handle_require(self, name, node, require, dep_graph, check_updates, update,
                         remotes, processed_profile, new_reqs, new_options):
+        # Handle a requirement of a node. There are 2 possibilities
+        #    node -(require)-> new_node (creates a new node in the graph)
+        #    node -(require)-> previous (creates a diamond with a previously existing node)
 
+        # If the required is found in the node ancestors a loop is being closed
+        # TODO: allow bootstrapping, use references instead of names
         if name in node.ancestors or name == node.name:
             raise ConanException("Loop detected: '%s' requires '%s' which is an ancestor too"
                                  % (node.ref, require.ref))
 
+        # If the requirement is found in the node public dependencies, it is a diamond
         previous = node.public_deps.get(name)
         previous_closure = node.public_closure.get(name)
+        # build_requires and private will create a new node if it is not in the current closure
         if not previous or ((require.build_require or require.private) and not previous_closure):
-            # new node, must be added and expanded
-            # node -> new_node
-            new_node = self._create_new_node(node, dep_graph, require, name,
-                                             check_updates, update, remotes,
-                                             processed_profile)
+            # new node, must be added and expanded (node -> new_node)
+            new_node = self._create_new_node(node, dep_graph, require, name, check_updates, update,
+                                             remotes, processed_profile)
 
             # The closure of a new node starts with just itself
             new_node.public_closure = OrderedDict([(new_node.ref.name, new_node)])
-            node.public_closure[name] = new_node
-            new_node.inverse_closure.add(node)
-            node.public_deps[new_node.name] = new_node
-
-            # If the parent node is a build-require, this new node will be a build-require
-            # If the requirement is a build-require, this node will also be a build-require
-            new_node.build_require = node.build_require or require.build_require
-            # New nodes will inherit the private property of its ancestor
-            new_node.private = node.private or require.private
+            # The new created node is connected to the parent one
+            node.connect_closure(new_node)
+
             if require.private or require.build_require:
-                # If the requirement is private (or build_require), a new public scope is defined
+                # If the requirement is private (or build_require), a new public_deps is defined
+                # the new_node doesn't propagate downstream the "node" consumer, so its public_deps
+                # will be a copy of the node.public_closure, i.e. it can only cause conflicts in the
+                # new_node.public_closure.
                 new_node.public_deps = node.public_closure.copy()
                 new_node.public_deps[name] = new_node
             else:
+                # Normal requires propagate and can conflict with the parent "node.public_deps" too
                 new_node.public_deps = node.public_deps.copy()
                 new_node.public_deps[name] = new_node
 
-                # Update the closure of each dependent
+                # All the dependents of "node" are also connected now to "new_node"
                 for dep_node in node.inverse_closure:
-                    dep_node.public_closure[new_node.name] = new_node
-                    new_node.inverse_closure.add(dep_node)
-                    dep_node.public_deps[new_node.name] = new_node
-
-            # RECURSION!
-            self._load_deps(dep_graph, new_node, new_reqs, node.ref,
-                            new_options, check_updates, update,
-                            remotes, processed_profile)
+                    dep_node.connect_closure(new_node)
+
+            # RECURSION, keep expanding (depth-first) the new node
+            self._load_deps(dep_graph, new_node, new_reqs, node.ref, new_options, check_updates,
+                            update, remotes, processed_profile)
         else:  # a public node already exist with this name
-            # This is closing a diamond, the node is existing in the scope
+            # This is closing a diamond, the node already exists and is reachable
             alias_ref = dep_graph.aliased.get(require.ref)
             # Necessary to make sure that it is pointing to the correct aliased
             if alias_ref:
                 require.ref = alias_ref
-            conflict = self._conflicting_references(previous.ref, require.ref)
-            if conflict == REVISION_CONFLICT:  # Revisions conflict
-                raise ConanException("Conflict in %s\n"
-                                     "    Different revisions of %s has been requested"
-                                     % (node.ref, require.ref))
-            elif conflict == REFERENCE_CONFLICT:
-                raise ConanException("Conflict in %s\n"
-                                     "    Requirement %s conflicts with already defined %s\n"
-                                     "    To change it, override it in your base requirements"
-                                     % (node.ref, require.ref, previous.ref))
+            # As we are closing a diamond, there can be conflicts. This will raise if conflicts
+            self._conflicting_references(previous.ref, require.ref, node.ref)
 
-            # Add current ancestors to the previous node
+            # Add current ancestors to the previous node and upstream deps
             union = node.ancestors.union([node.name])
-            for prev_node in previous.public_closure.values():
-                prev_node.ancestors.update(union)
+            for n in previous.public_closure.values():
+                n.ancestors.update(union)
+
+            # Even if it was in private scope, if it is reached via a public require
+            # the previous node and its upstream becomes public
             if previous.private and not require.private:
                 previous.make_public()
 
-            node.public_closure[name] = previous
-            previous.inverse_closure.add(node)
-            node.public_deps[name] = previous
+            node.connect_closure(previous)
             dep_graph.add_edge(node, previous, require.private, require.build_require)
-            # Update the closure of each dependent
+            # All the upstream dependencies (public_closure) of the previously existing node
+            # now will be also connected to the node and to all its dependants
             for name, n in previous.public_closure.items():
                 if n.build_require or n.private:
                     continue
-                node.public_closure[name] = n
-                n.inverse_closure.add(node)
+                node.connect_closure(n)
                 for dep_node in node.inverse_closure:
-                    dep_node.public_closure[name] = n
-                    dep_node.public_deps[name] = n
-                    n.inverse_closure.add(dep_node)
+                    dep_node.connect_closure(n)
 
-            # RECURSION!
+            # Recursion is only necessary if the inputs conflict with the current "previous"
+            # configuration of upstream versions and options
             if self._recurse(previous.public_closure, new_reqs, new_options):
-                self._load_deps(dep_graph, previous, new_reqs, node.ref,
-                                new_options, check_updates, update,
-                                remotes, processed_profile)
+                self._load_deps(dep_graph, previous, new_reqs, node.ref, new_options, check_updates,
+                                update, remotes, processed_profile)
 
     @staticmethod
-    def _conflicting_references(previous_ref, new_ref):
+    def _conflicting_references(previous_ref, new_ref, consumer_ref=None):
         if previous_ref.copy_clear_rev() != new_ref.copy_clear_rev():
-            return REFERENCE_CONFLICT
+            if consumer_ref:
+                raise ConanException("Conflict in %s\n"
+                                     "    Requirement %s conflicts with already defined %s\n"
+                                     "    To change it, override it in your base requirements"
+                                     % (consumer_ref, new_ref, previous_ref))
+            return True
         # Computed node, if is Editable, has revision=None
         # If new_ref.revision is None we cannot assume any conflict, the user hasn't specified
         # a revision, so it's ok any previous_ref
         if previous_ref.revision and new_ref.revision and previous_ref.revision != new_ref.revision:
-            return REVISION_CONFLICT
+            if consumer_ref:
+                raise ConanException("Conflict in %s\n"
+                                     "    Different revisions of %s has been requested"
+                                     % (consumer_ref, new_ref))
+            return True
         return False
 
     def _recurse(self, closure, new_reqs, new_options):
@@ -341,8 +336,15 @@ def _create_new_node(self, current_node, dep_graph, requirement, name_req,
         new_node.revision_pinned = requirement.ref.revision is not None
         new_node.recipe = recipe_status
         new_node.remote = remote
+        # Ancestors are a copy of the parent, plus the parent itself
         new_node.ancestors = current_node.ancestors.copy()
         new_node.ancestors.add(current_node.name)
+        # build-requires and private affect transitively. If "node" is already
+        # a build_require or a private one, its requirements will inherit that property
+        # Or if the require specify that property, then it will get it too
+        new_node.build_require = current_node.build_require or requirement.build_require
+        new_node.private = current_node.private or requirement.private
+
         dep_graph.add_node(new_node)
         dep_graph.add_edge(current_node, new_node, requirement.private, requirement.build_require)
         return new_node


Test patch: