Repository: conan-io/conan

Issue:
Enhance graph dependency resolution speed
Hello,

I have a big dependency graph that is managed in our CI system. About a hundred packages are handled and depends on each other by using version range like `[^1.66.0]` to keep, according to semver, the last backward compatible package.

The fact is that as all packages requirements are expressed that way.
I observed that conan, for each package, make a query to resolve the range and that can take a lot of time.

Observed times are with conan <= 1.4 a `conan info . -bo ... -bo ... [...]` on the graph would take about fifteen minutes (depending on the number of `-bo` that are passed).
On conan 1.6, it is twice as before.

As a way to enhance the resolution, can conan create a cache so they are resolved only once and future calls to the same range will be answered quickly ?


Discussion:
Hi @dkgs 

Thanks for the feedback. Yes, it might be an effect of how the graph is computed now, with build-requires being duplicated nodes (build-requires are private to the packages that require them, same as private). And we are aware that resolving ranges is slow, as it involves searching in the remote servers. We will try to have a look at this and see if the performance could be improved.

Resolving patch:
diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py
--- a/conans/client/graph/graph_builder.py
+++ b/conans/client/graph/graph_builder.py
@@ -50,7 +50,7 @@ def _resolve_deps(self, node, aliased, update, remote_name):
 
         # After resolving ranges,
         for req in conanfile.requires.values():
-            alias = aliased.get(req.conan_reference, None)
+            alias = aliased.get(req.conan_reference)
             if alias:
                 req.conan_reference = alias
 
diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py
--- a/conans/client/graph/range_resolver.py
+++ b/conans/client/graph/range_resolver.py
@@ -27,6 +27,7 @@ def __init__(self, output, client_cache, remote_search):
         self._output = output
         self._client_cache = client_cache
         self._remote_search = remote_search
+        self._cached_remote_found = {}
 
     def resolve(self, require, base_conanref, update, remote_name):
         version_range = require.version_range
@@ -69,13 +70,16 @@ def resolve(self, require, base_conanref, update, remote_name):
     def _resolve_local(self, search_ref, version_range):
         local_found = search_recipes(self._client_cache, search_ref)
         if local_found:
-            resolved_version = self._resolve_version(version_range, local_found)
-            if resolved_version:
-                return resolved_version
+            return self._resolve_version(version_range, local_found)
 
     def _resolve_remote(self, search_ref, version_range, remote_name):
+        remote_cache = self._cached_remote_found.setdefault(remote_name, {})
         # We should use ignorecase=False, we want the exact case!
-        remote_found = self._remote_search.search_remotes(search_ref, remote_name)
+        remote_found = remote_cache.get(search_ref)
+        if remote_found is None:
+            remote_found = self._remote_search.search_remotes(search_ref, remote_name)
+            # Empty list, just in case it returns None
+            remote_cache[search_ref] = remote_found or []
         if remote_found:
             return self._resolve_version(version_range, remote_found)
 


Test patch:
diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py
--- a/conans/test/model/version_ranges_test.py
+++ b/conans/test/model/version_ranges_test.py
@@ -1,6 +1,6 @@
 import os
 import unittest
-from collections import namedtuple
+from collections import namedtuple, Counter
 
 from conans.test.utils.tools import TestBufferConanOutput
 from conans.paths import CONANFILE, SimplePaths
@@ -141,8 +141,10 @@ def _get_edges(graph):
 class MockSearchRemote(object):
     def __init__(self, packages=None):
         self.packages = packages or []
+        self.count = Counter()
 
     def search_remotes(self, pattern, ignorecase):  # @UnusedVariable
+        self.count[pattern] += 1
         return self.packages
 
 
@@ -168,9 +170,9 @@ class SayConan(ConanFile):
             say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
             self.retriever.conan(say_ref, say_content)
 
-    def root(self, content):
+    def root(self, content, update=False):
         root_conan = self.retriever.root(content)
-        deps_graph = self.builder.load_graph(root_conan, False, False, None)
+        deps_graph = self.builder.load_graph(root_conan, update, update, None)
         return deps_graph
 
     def test_local_basic(self):
@@ -205,7 +207,63 @@ def test_remote_basic(self):
             say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
             remote_packages.append(say_ref)
         self.remote_search.packages = remote_packages
-        self.test_local_basic()
+        for expr, solution in [(">0.0", "2.2.1"),
+                               (">0.1,<1", "0.3"),
+                               (">0.1,<1||2.1", "2.1"),
+                               ("", "2.2.1"),
+                               ("~0", "0.3"),
+                               ("~=1", "1.2.1"),
+                               ("~1.1", "1.1.2"),
+                               ("~=2", "2.2.1"),
+                               ("~=2.1", "2.1"),
+                               ]:
+            deps_graph = self.root(hello_content % expr, update=True)
+            self.assertEqual(self.remote_search.count, {'Say/*@memsharded/testing': 1})
+            self.assertEqual(2, len(deps_graph.nodes))
+            hello = _get_nodes(deps_graph, "Hello")[0]
+            say = _get_nodes(deps_graph, "Say")[0]
+            self.assertEqual(_get_edges(deps_graph), {Edge(hello, say)})
+
+            self.assertEqual(hello.conan_ref, None)
+            conanfile = hello.conanfile
+            self.assertEqual(conanfile.version, "1.2")
+            self.assertEqual(conanfile.name, "Hello")
+            say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % solution)
+            self.assertEqual(conanfile.requires, Requirements(str(say_ref)))
+
+    def test_remote_optimized(self):
+        self.resolver._local_search = None
+        remote_packages = []
+        for v in ["0.1", "0.2", "0.3", "1.1", "1.1.2", "1.2.1", "2.1", "2.2.1"]:
+            say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
+            remote_packages.append(say_ref)
+        self.remote_search.packages = remote_packages
+
+        dep_content = """from conans import ConanFile
+class Dep1Conan(ConanFile):
+    requires = "Say/[%s]@memsharded/testing"
+"""
+        dep_ref = ConanFileReference.loads("Dep1/0.1@memsharded/testing")
+        self.retriever.conan(dep_ref, dep_content % ">=0.1")
+        dep_ref = ConanFileReference.loads("Dep2/0.1@memsharded/testing")
+        self.retriever.conan(dep_ref, dep_content % ">=0.1")
+
+        hello_content = """from conans import ConanFile
+class HelloConan(ConanFile):
+    name = "Hello"
+    requires = "Dep1/0.1@memsharded/testing", "Dep2/0.1@memsharded/testing"
+"""
+        deps_graph = self.root(hello_content, update=True)
+        self.assertEqual(4, len(deps_graph.nodes))
+        hello = _get_nodes(deps_graph, "Hello")[0]
+        say = _get_nodes(deps_graph, "Say")[0]
+        dep1 = _get_nodes(deps_graph, "Dep1")[0]
+        dep2 = _get_nodes(deps_graph, "Dep2")[0]
+        self.assertEqual(_get_edges(deps_graph), {Edge(hello, dep1), Edge(hello, dep2),
+                                                  Edge(dep1, say), Edge(dep2, say)})
+
+        # Most important check: counter of calls to remote
+        self.assertEqual(self.remote_search.count, {'Say/*@memsharded/testing': 1})
 
     @parameterized.expand([("", "0.3", None, None),
                            ('"Say/1.1@memsharded/testing"', "1.1", False, False),