Repository: conan-io/conan

Issue:
scm query
Is there a reason the .git folder is excluded from the copy that occurs? Would it be possible to ensure that it isn't ignored?

I've got a tool I'm using in parts of my conanfile that requires being able to query the git repo. I think it would make sense for the .git folder to be included in the copy to make the export from source closer to what happens on a machine that pulls the recipe and tries to build a non-existant hash.

https://github.com/conan-io/conan/blob/94222228824c90f482cd56769667d1f5c42cca8e/conans/client/tools/scm.py#L99

Thanks!
Matt

Discussion:

Patch:
diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py
index c3e671d0e..db30049d5 100644
--- a/conans/client/graph/graph_builder.py
+++ b/conans/client/graph/graph_builder.py
@@ -50,7 +50,7 @@ class DepsGraphBuilder(object):
 
         # After resolving ranges,
         for req in conanfile.requires.values():
-            alias = aliased.get(req.conan_reference, None)
+            alias = aliased.get(req.conan_reference)
             if alias:
                 req.conan_reference = alias
 
diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py
index c92c5fe23..45c488f2b 100644
--- a/conans/client/graph/range_resolver.py
+++ b/conans/client/graph/range_resolver.py
@@ -27,6 +27,7 @@ class RangeResolver(object):
         self._output = output
         self._client_cache = client_cache
         self._remote_search = remote_search
+        self._cached_remote_found = {}
 
     def resolve(self, require, base_conanref, update, remote_name):
         version_range = require.version_range
@@ -69,13 +70,16 @@ class RangeResolver(object):
     def _resolve_local(self, search_ref, version_range):
         local_found = search_recipes(self._client_cache, search_ref)
         if local_found:
-            resolved_version = self._resolve_version(version_range, local_found)
-            if resolved_version:
-                return resolved_version
+            return self._resolve_version(version_range, local_found)
 
     def _resolve_remote(self, search_ref, version_range, remote_name):
+        remote_cache = self._cached_remote_found.setdefault(remote_name, {})
         # We should use ignorecase=False, we want the exact case!
-        remote_found = self._remote_search.search_remotes(search_ref, remote_name)
+        remote_found = remote_cache.get(search_ref)
+        if remote_found is None:
+            remote_found = self._remote_search.search_remotes(search_ref, remote_name)
+            # Empty list, just in case it returns None
+            remote_cache[search_ref] = remote_found or []
         if remote_found:
             return self._resolve_version(version_range, remote_found)
 
diff --git a/conans/client/tools/scm.py b/conans/client/tools/scm.py
index d43c385e5..373c578f7 100644
--- a/conans/client/tools/scm.py
+++ b/conans/client/tools/scm.py
@@ -47,7 +47,7 @@ class Git(object):
     def _configure_ssl_verify(self):
         return self.run("config http.sslVerify %s" % ("true" if self._verify_ssl else "false"))
 
-    def clone(self, url, branch=None, submodule=None):
+    def clone(self, url, branch=None):
         url = self.get_url_with_credentials(url)
         if os.path.exists(url):
             url = url.replace("\\", "/")  # Windows local directory
@@ -67,6 +67,12 @@ class Git(object):
             output = self.run('clone "%s" . %s' % (url, branch_cmd))
             output += self._configure_ssl_verify()
 
+        return output
+
+    def checkout(self, element, submodule=None):
+        self._check_git_repo()
+        output = self.run('checkout "%s"' % element)
+
         if submodule:
             if submodule == "shallow":
                 output += self.run("submodule sync")
@@ -77,13 +83,8 @@ class Git(object):
             else:
                 raise ConanException("Invalid 'submodule' attribute value in the 'scm'. "
                                      "Unknown value '%s'. Allowed values: ['shallow', 'recursive']" % submodule)
-
-        return output
-
-    def checkout(self, element):
-        self._check_git_repo()
         # Element can be a tag, branch or commit
-        return self.run('checkout "%s"' % element)
+        return output
 
     def excluded_files(self):
         try:
@@ -96,7 +97,6 @@ class Git(object):
             tmp = grep_stdout.splitlines()
         except CalledProcessError:
             tmp = []
-        tmp.append(".git")
         return tmp
 
     def get_remote_url(self, remote_name=None):
diff --git a/conans/model/scm.py b/conans/model/scm.py
index e3a47e2e2..e7ff64aca 100644
--- a/conans/model/scm.py
+++ b/conans/model/scm.py
@@ -65,10 +65,10 @@ class SCM(object):
         return self.repo.excluded_files()
 
     def clone(self):
-        return self.repo.clone(self._data.url, submodule=self._data.submodule)
+        return self.repo.clone(self._data.url)
 
     def checkout(self):
-        return self.repo.checkout(self._data.revision)
+        return self.repo.checkout(self._data.revision, submodule=self._data.submodule)
 
     def get_remote_url(self):
         return self.repo.get_remote_url()


Test patch:
diff --git a/conans/test/functional/scm_test.py b/conans/test/functional/scm_test.py
index 8564b0789..48aeacff2 100644
--- a/conans/test/functional/scm_test.py
+++ b/conans/test/functional/scm_test.py
@@ -184,7 +184,7 @@ class ConanLib(ConanFile):
         bf = self.client.client_cache.build(pref)
         self.assertTrue(os.path.exists(os.path.join(bf, "myfile.txt")))
         self.assertTrue(os.path.exists(os.path.join(bf, "myfile")))
-        self.assertFalse(os.path.exists(os.path.join(bf, ".git")))
+        self.assertTrue(os.path.exists(os.path.join(bf, ".git")))
         self.assertFalse(os.path.exists(os.path.join(bf, "ignored.pyc")))
 
     def test_local_source(self):
diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py
index e42560f05..3e0122605 100644
--- a/conans/test/model/version_ranges_test.py
+++ b/conans/test/model/version_ranges_test.py
@@ -1,6 +1,6 @@
 import os
 import unittest
-from collections import namedtuple
+from collections import namedtuple, Counter
 
 from conans.test.utils.tools import TestBufferConanOutput
 from conans.paths import CONANFILE, SimplePaths
@@ -141,8 +141,10 @@ def _get_edges(graph):
 class MockSearchRemote(object):
     def __init__(self, packages=None):
         self.packages = packages or []
+        self.count = Counter()
 
     def search_remotes(self, pattern, ignorecase):  # @UnusedVariable
+        self.count[pattern] += 1
         return self.packages
 
 
@@ -168,9 +170,9 @@ class SayConan(ConanFile):
             say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
             self.retriever.conan(say_ref, say_content)
 
-    def root(self, content):
+    def root(self, content, update=False):
         root_conan = self.retriever.root(content)
-        deps_graph = self.builder.load_graph(root_conan, False, False, None)
+        deps_graph = self.builder.load_graph(root_conan, update, update, None)
         return deps_graph
 
     def test_local_basic(self):
@@ -205,7 +207,63 @@ class SayConan(ConanFile):
             say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
             remote_packages.append(say_ref)
         self.remote_search.packages = remote_packages
-        self.test_local_basic()
+        for expr, solution in [(">0.0", "2.2.1"),
+                               (">0.1,<1", "0.3"),
+                               (">0.1,<1||2.1", "2.1"),
+                               ("", "2.2.1"),
+                               ("~0", "0.3"),
+                               ("~=1", "1.2.1"),
+                               ("~1.1", "1.1.2"),
+                               ("~=2", "2.2.1"),
+                               ("~=2.1", "2.1"),
+                               ]:
+            deps_graph = self.root(hello_content % expr, update=True)
+            self.assertEqual(self.remote_search.count, {'Say/*@memsharded/testing': 1})
+            self.assertEqual(2, len(deps_graph.nodes))
+            hello = _get_nodes(deps_graph, "Hello")[0]
+            say = _get_nodes(deps_graph, "Say")[0]
+            self.assertEqual(_get_edges(deps_graph), {Edge(hello, say)})
+
+            self.assertEqual(hello.conan_ref, None)
+            conanfile = hello.conanfile
+            self.assertEqual(conanfile.version, "1.2")
+            self.assertEqual(conanfile.name, "Hello")
+            say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % solution)
+            self.assertEqual(conanfile.requires, Requirements(str(say_ref)))
+
+    def test_remote_optimized(self):
+        self.resolver._local_search = None
+        remote_packages = []
+        for v in ["0.1", "0.2", "0.3", "1.1", "1.1.2", "1.2.1", "2.1", "2.2.1"]:
+            say_ref = ConanFileReference.loads("Say/%s@memsharded/testing" % v)
+            remote_packages.append(say_ref)
+        self.remote_search.packages = remote_packages
+
+        dep_content = """from conans import ConanFile
+class Dep1Conan(ConanFile):
+    requires = "Say/[%s]@memsharded/testing"
+"""
+        dep_ref = ConanFileReference.loads("Dep1/0.1@memsharded/testing")
+        self.retriever.conan(dep_ref, dep_content % ">=0.1")
+        dep_ref = ConanFileReference.loads("Dep2/0.1@memsharded/testing")
+        self.retriever.conan(dep_ref, dep_content % ">=0.1")
+
+        hello_content = """from conans import ConanFile
+class HelloConan(ConanFile):
+    name = "Hello"
+    requires = "Dep1/0.1@memsharded/testing", "Dep2/0.1@memsharded/testing"
+"""
+        deps_graph = self.root(hello_content, update=True)
+        self.assertEqual(4, len(deps_graph.nodes))
+        hello = _get_nodes(deps_graph, "Hello")[0]
+        say = _get_nodes(deps_graph, "Say")[0]
+        dep1 = _get_nodes(deps_graph, "Dep1")[0]
+        dep2 = _get_nodes(deps_graph, "Dep2")[0]
+        self.assertEqual(_get_edges(deps_graph), {Edge(hello, dep1), Edge(hello, dep2),
+                                                  Edge(dep1, say), Edge(dep2, say)})
+
+        # Most important check: counter of calls to remote
+        self.assertEqual(self.remote_search.count, {'Say/*@memsharded/testing': 1})
 
     @parameterized.expand([("", "0.3", None, None),
                            ('"Say/1.1@memsharded/testing"', "1.1", False, False),
diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py
index 21a951435..86576451b 100644
--- a/conans/test/util/tools_test.py
+++ b/conans/test/util/tools_test.py
@@ -1125,7 +1125,7 @@ class GitToolTest(unittest.TestCase):
     def test_clone_submodule_git(self):
         subsubmodule, _ = create_local_git_repo({"subsubmodule": "contents"})
         submodule, _ = create_local_git_repo({"submodule": "contents"}, submodules=[subsubmodule])
-        path, _ = create_local_git_repo({"myfile": "contents"}, submodules=[submodule])
+        path, commit = create_local_git_repo({"myfile": "contents"}, submodules=[submodule])
 
         def _create_paths():
             tmp = temp_folder()
@@ -1147,13 +1147,15 @@ class GitToolTest(unittest.TestCase):
         # Check invalid value
         tmp, submodule_path, subsubmodule_path = _create_paths()
         git = Git(tmp)
+        git.clone(path)
         with self.assertRaisesRegexp(ConanException, "Invalid 'submodule' attribute value in the 'scm'."):
-            git.clone(path, submodule="invalid")
+            git.checkout(commit, submodule="invalid")
 
         # Check shallow
         tmp, submodule_path, subsubmodule_path = _create_paths()
         git = Git(tmp)
-        git.clone(path, submodule="shallow")
+        git.clone(path)
+        git.checkout(commit, submodule="shallow")
         self.assertTrue(os.path.exists(os.path.join(tmp, "myfile")))
         self.assertTrue(os.path.exists(os.path.join(submodule_path, "submodule")))
         self.assertFalse(os.path.exists(os.path.join(subsubmodule_path, "subsubmodule")))
@@ -1161,7 +1163,8 @@ class GitToolTest(unittest.TestCase):
         # Check recursive
         tmp, submodule_path, subsubmodule_path = _create_paths()
         git = Git(tmp)
-        git.clone(path, submodule="recursive")
+        git.clone(path)
+        git.checkout(commit, submodule="recursive")
         self.assertTrue(os.path.exists(os.path.join(tmp, "myfile")))
         self.assertTrue(os.path.exists(os.path.join(submodule_path, "submodule")))
         self.assertTrue(os.path.exists(os.path.join(subsubmodule_path, "subsubmodule")))