Repository: conan-io/conan
Pull request #16342 (2024-05-26 22:04:06)

proposal for user/channel in local-recipes-index
Changelog: Fix: Allow "local-recipes-index" to ``conan list`` packages with custom ``user/channel``
Docs: Omit

Close https://github.com/conan-io/conan/issues/16331



Patch:
diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py
index 77d8c6df8..e8504efc1 100644
--- a/conan/api/subapi/cache.py
+++ b/conan/api/subapi/cache.py
@@ -177,7 +177,10 @@ class CacheAPI:
         for ref, ref_bundle in package_list.refs().items():
             ref.timestamp = revision_timestamp_now()
             ref_bundle["timestamp"] = ref.timestamp
-            recipe_layout = cache.get_or_create_ref_layout(ref)  # DB folder entry
+            try:
+                recipe_layout = cache.recipe_layout(ref)
+            except ConanException:
+                recipe_layout = cache.create_ref_layout(ref)  # new DB folder entry
             recipe_folder = ref_bundle["recipe_folder"]
             rel_path = os.path.relpath(recipe_layout.base_folder, cache_folder)
             rel_path = rel_path.replace("\\", "/")
@@ -187,7 +190,11 @@ class CacheAPI:
             for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items():
                 pref.timestamp = revision_timestamp_now()
                 pref_bundle["timestamp"] = pref.timestamp
-                pkg_layout = cache.get_or_create_pkg_layout(pref)  # DB Folder entry
+                try:
+                    pkg_layout = cache.pkg_layout(pref)
+                except ConanException:
+                    pkg_layout = cache.create_pkg_layout(pref)  # DB Folder entry
+                # FIXME: This is not taking into account the existence of previous package
                 unzipped_pkg_folder = pref_bundle["package_folder"]
                 out.info(f"Restore: {pref} in {unzipped_pkg_folder}")
                 # If the DB folder entry is different to the disk unzipped one, we need to move it
@@ -203,6 +210,7 @@ class CacheAPI:
                     pref_bundle["package_folder"] = db_pkg_folder
                 unzipped_metadata_folder = pref_bundle.get("metadata_folder")
                 if unzipped_metadata_folder:
+                    # FIXME: Restore metadata is not incremental, but destructive
                     out.info(f"Restore: {pref} metadata in {unzipped_metadata_folder}")
                     db_metadata_folder = os.path.relpath(pkg_layout.metadata(), cache_folder)
                     db_metadata_folder = db_metadata_folder.replace("\\", "/")
diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py
index edb2bb94a..92af0a603 100644
--- a/conan/internal/cache/cache.py
+++ b/conan/internal/cache/cache.py
@@ -8,8 +8,7 @@ from conan.internal.cache.conan_reference_layout import RecipeLayout, PackageLay
 # TODO: Random folders are no longer accessible, how to get rid of them asap?
 # TODO: We need the workflow to remove existing references.
 from conan.internal.cache.db.cache_database import CacheDatabase
-from conans.errors import ConanReferenceAlreadyExistsInDB, ConanReferenceDoesNotExistInDB, \
-    ConanException
+from conans.errors import ConanReferenceAlreadyExistsInDB, ConanException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
 from conans.util.dates import revision_timestamp_now
@@ -138,31 +137,29 @@ class PkgCache:
         # we use abspath to convert cache forward slash in Windows to backslash
         return PackageLayout(pref, os.path.abspath(os.path.join(self._base_folder, pref_path)))
 
-    def get_or_create_ref_layout(self, ref: RecipeReference):
-        """ called by RemoteManager.get_recipe()
+    def create_ref_layout(self, ref: RecipeReference):
+        """ called exclusively by:
+        - RemoteManager.get_recipe()
+        - cache restore
         """
-        try:
-            return self.recipe_layout(ref)
-        except ConanReferenceDoesNotExistInDB:
-            assert ref.revision, "Recipe revision must be known to create the package layout"
-            reference_path = self._get_path(ref)
-            self._db.create_recipe(reference_path, ref)
-            self._create_path(reference_path, remove_contents=False)
-            return RecipeLayout(ref, os.path.join(self._base_folder, reference_path))
-
-    def get_or_create_pkg_layout(self, pref: PkgReference):
-        """ called by RemoteManager.get_package() and  BinaryInstaller
+        assert ref.revision, "Recipe revision must be known to create the package layout"
+        reference_path = self._get_path(ref)
+        self._db.create_recipe(reference_path, ref)
+        self._create_path(reference_path, remove_contents=False)
+        return RecipeLayout(ref, os.path.join(self._base_folder, reference_path))
+
+    def create_pkg_layout(self, pref: PkgReference):
+        """ called by:
+         - RemoteManager.get_package()
+         - cacje restpre
         """
-        try:
-            return self.pkg_layout(pref)
-        except ConanReferenceDoesNotExistInDB:
-            assert pref.ref.revision, "Recipe revision must be known to create the package layout"
-            assert pref.package_id, "Package id must be known to create the package layout"
-            assert pref.revision, "Package revision should be known to create the package layout"
-            package_path = self._get_path_pref(pref)
-            self._db.create_package(package_path, pref, None)
-            self._create_path(package_path, remove_contents=False)
-            return PackageLayout(pref, os.path.join(self._base_folder, package_path))
+        assert pref.ref.revision, "Recipe revision must be known to create the package layout"
+        assert pref.package_id, "Package id must be known to create the package layout"
+        assert pref.revision, "Package revision should be known to create the package layout"
+        package_path = self._get_path_pref(pref)
+        self._db.create_package(package_path, pref, None)
+        self._create_path(package_path, remove_contents=False)
+        return PackageLayout(pref, os.path.join(self._base_folder, package_path))
 
     def update_recipe_timestamp(self, ref: RecipeReference):
         """ when the recipe already exists in cache, but we get a new timestamp from a server
diff --git a/conan/internal/cache/conan_reference_layout.py b/conan/internal/cache/conan_reference_layout.py
index 592c03c68..3a4aab9b0 100644
--- a/conan/internal/cache/conan_reference_layout.py
+++ b/conan/internal/cache/conan_reference_layout.py
@@ -81,14 +81,6 @@ class RecipeLayout(LayoutBase):
         src_folder = self.source()
         rmdir(src_folder)
 
-    def export_remove(self):
-        export_folder = self.export()
-        rmdir(export_folder)
-        export_src_folder = os.path.join(self._base_folder, EXPORT_SRC_FOLDER)
-        rmdir(export_src_folder)
-        download_export = self.download_export()
-        rmdir(download_export)
-
 
 class PackageLayout(LayoutBase):
 
diff --git a/conans/client/installer.py b/conans/client/installer.py
index 7c05e2724..5f62d1527 100644
--- a/conans/client/installer.py
+++ b/conans/client/installer.py
@@ -304,30 +304,29 @@ class BinaryInstaller:
 
         pref = PkgReference(install_reference.ref, package.package_id, package.prev)
 
-        if pref.revision is None:
-            assert package.binary == BINARY_BUILD
-            package_layout = self._cache.create_build_pkg_layout(pref)
-        else:
-            package_layout = self._cache.get_or_create_pkg_layout(pref)
-
         if package.binary == BINARY_BUILD:
+            assert pref.revision is None
             ConanOutput()\
                 .subtitle(f"Installing package {pref.ref} ({handled_count} of {total_count})")
             ConanOutput(scope=str(pref.ref))\
                 .highlight("Building from source")\
                 .info(f"Package {pref}")
+            package_layout = self._cache.create_build_pkg_layout(pref)
             self._handle_node_build(package, package_layout)
             # Just in case it was recomputed
             package.package_id = package.nodes[0].pref.package_id  # Just in case it was recomputed
             package.prev = package.nodes[0].pref.revision
             package.binary = package.nodes[0].binary
             pref = PkgReference(install_reference.ref, package.package_id, package.prev)
-        elif package.binary == BINARY_CACHE:
-            node = package.nodes[0]
-            pref = node.pref
-            self._cache.update_package_lru(pref)
-            assert node.prev, "PREV for %s is None" % str(pref)
-            node.conanfile.output.success(f'Already installed! ({handled_count} of {total_count})')
+        else:
+            assert pref.revision is not None
+            package_layout = self._cache.pkg_layout(pref)
+            if package.binary == BINARY_CACHE:
+                node = package.nodes[0]
+                pref = node.pref
+                self._cache.update_package_lru(pref)
+                assert node.prev, "PREV for %s is None" % str(pref)
+                node.conanfile.output.success(f'Already installed! ({handled_count} of {total_count})')
 
         # Make sure that all nodes with same pref compute package_info()
         pkg_folder = package_layout.package()
diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py
index 9827b83d7..51ce2e2d1 100644
--- a/conans/client/remote_manager.py
+++ b/conans/client/remote_manager.py
@@ -49,8 +49,7 @@ class RemoteManager:
         assert ref.revision, "get_recipe without revision specified"
         assert ref.timestamp, "get_recipe without ref.timestamp specified"
 
-        layout = self._cache.get_or_create_ref_layout(ref)
-        layout.export_remove()
+        layout = self._cache.create_ref_layout(ref)
 
         export_folder = layout.export()
         local_folder_remote = self._local_folder_remote(remote)
@@ -132,8 +131,7 @@ class RemoteManager:
 
         assert pref.revision is not None
 
-        pkg_layout = self._cache.get_or_create_pkg_layout(pref)
-        pkg_layout.package_remove()  # Remove first the destination folder
+        pkg_layout = self._cache.create_pkg_layout(pref)
         with pkg_layout.set_dirty_context_manager():
             self._get_package(pkg_layout, pref, remote, output, metadata)
 
diff --git a/conans/client/rest_client_local_recipe_index.py b/conans/client/rest_client_local_recipe_index.py
index 36d3b516e..041406d50 100644
--- a/conans/client/rest_client_local_recipe_index.py
+++ b/conans/client/rest_client_local_recipe_index.py
@@ -10,6 +10,7 @@ from conan.api.model import LOCAL_RECIPES_INDEX
 from conan.api.output import ConanOutput
 from conan.internal.cache.home_paths import HomePaths
 from conans.client.cmd.export import cmd_export
+from conans.client.loader import ConanFileLoader
 from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException
 from conans.model.conf import ConfDefinition
 from conans.model.recipe_ref import RecipeReference
@@ -190,6 +191,8 @@ class _LocalRecipesIndexLayout:
         recipes.sort()
         ret = []
         excluded = set()
+
+        loader = ConanFileLoader(None)
         for r in recipes:
             if not fnmatch(r, name_pattern):
                 continue
@@ -207,10 +210,18 @@ class _LocalRecipesIndexLayout:
                 # This check can be removed after compatibility with 2.0
                 conanfile = os.path.join(recipes_dir, r, subfolder, "conanfile.py")
                 conanfile_content = load(conanfile)
+
                 if "from conans" in conanfile_content or "import conans" in conanfile_content:
                     excluded.add(r)
                     continue
-                ret.append(RecipeReference.loads(ref))
+                ref = RecipeReference.loads(ref)
+                try:
+                    recipe = loader.load_basic(conanfile)
+                    ref.user = recipe.user
+                    ref.channel = recipe.channel
+                except Exception as e:
+                    ConanOutput().warning(f"Couldn't load recipe {conanfile}: {e}")
+                ret.append(ref)
         if excluded:
             ConanOutput().warning(f"Excluding recipes not Conan 2.0 ready: {', '.join(excluded)}")
         return ret


Test patch:
diff --git a/test/integration/graph/core/graph_manager_base.py b/test/integration/graph/core/graph_manager_base.py
index 6ff38f269..7f4ac1f73 100644
--- a/test/integration/graph/core/graph_manager_base.py
+++ b/test/integration/graph/core/graph_manager_base.py
@@ -50,7 +50,7 @@ class GraphManagerTest(unittest.TestCase):
     def _put_in_cache(self, ref, conanfile):
         ref = RecipeReference.loads("{}#123".format(ref))
         ref.timestamp = revision_timestamp_now()
-        layout = self.cache.get_or_create_ref_layout(ref)
+        layout = self.cache.create_ref_layout(ref)
         save(layout.conanfile(), str(conanfile))
         manifest = FileTreeManifest.create(layout.export())
         manifest.save(layout.export())
@@ -61,7 +61,7 @@ class GraphManagerTest(unittest.TestCase):
             ref = RecipeReference.loads(ref)
         ref = RecipeReference.loads(repr(ref) + "#{}".format(revision or 123))  # FIXME: Make access
         ref.timestamp = revision_timestamp_now()
-        recipe_layout = self.cache.get_or_create_ref_layout(ref)
+        recipe_layout = self.cache.create_ref_layout(ref)
         save(recipe_layout.conanfile(), str(test_conanfile))
         manifest = FileTreeManifest.create(recipe_layout.export())
         manifest.save(recipe_layout.export())
diff --git a/test/integration/graph/core/graph_manager_test.py b/test/integration/graph/core/graph_manager_test.py
index 290029814..bf620c4ca 100644
--- a/test/integration/graph/core/graph_manager_test.py
+++ b/test/integration/graph/core/graph_manager_test.py
@@ -349,24 +349,23 @@ class TestLinear(GraphManagerTest):
         _check_transitive(app, [(libb, True, True, False, False)])
         _check_transitive(libb, [(liba, False, False, True, True)])
 
-    def test_generic_build_require_adjust_run_with_package_type(self):
-        # FIXME: parametrize when superclass is not unittest
-        for package_type in ["application", "shared-library", "static-library",
-                             "header-library", "build-scripts", None]:
-            # app --br-> cmake (app)
-            self.recipe_conanfile("cmake/0.1", GenConanfile().with_package_type(package_type))
-            # build require with run=None by default
-            consumer = self.recipe_consumer("app/0.1", build_requires=["cmake/0.1"])
-
-            deps_graph = self.build_consumer(consumer)
-
-            self.assertEqual(2, len(deps_graph.nodes))
-            app = deps_graph.root
-            cmake = app.dependencies[0].dst
-
-            # node, headers, lib, build, run
-            run = package_type in ("application", "shared-library", "build-scripts")
-            _check_transitive(app, [(cmake, False, False, True, run)])
+    @parameterized.expand([("application",), ("shared-library",), ("static-library",),
+                           ("header-library",), ("build-scripts",), (None,)])
+    def test_generic_build_require_adjust_run_with_package_type(self, package_type):
+        # app --br-> cmake (app)
+        self.recipe_conanfile("cmake/0.1", GenConanfile().with_package_type(package_type))
+        # build require with run=None by default
+        consumer = self.recipe_consumer("app/0.1", build_requires=["cmake/0.1"])
+
+        deps_graph = self.build_consumer(consumer)
+
+        self.assertEqual(2, len(deps_graph.nodes))
+        app = deps_graph.root
+        cmake = app.dependencies[0].dst
+
+        # node, headers, lib, build, run
+        run = package_type in ("application", "shared-library", "build-scripts")
+        _check_transitive(app, [(cmake, False, False, True, run)])
 
     def test_direct_header_only(self):
         # app -> liba0.1 (header_only)
diff --git a/test/integration/remote/test_local_recipes_index.py b/test/integration/remote/test_local_recipes_index.py
index e96dddce3..2876832b6 100644
--- a/test/integration/remote/test_local_recipes_index.py
+++ b/test/integration/remote/test_local_recipes_index.py
@@ -270,6 +270,30 @@ class TestInstall:
         assert "zlib/0.1: Copied 1 file: patch1" in client.out
         assert "zlib/0.1: Apply patch (file): patches/patch1" in client.out
 
+    def test_export_user_channel(self):
+        folder = temp_folder()
+        recipes_folder = os.path.join(folder, "recipes")
+        zlib_config = textwrap.dedent("""
+            versions:
+              "0.1":
+                folder: all
+            """)
+        zlib = GenConanfile("zlib").with_class_attribute("user='myuser'")\
+                                   .with_class_attribute("channel='mychannel'")
+        conandata_yml = textwrap.dedent("""\
+            versions:
+              "0.1":
+            """)
+        save_files(recipes_folder, {"zlib/config.yml": zlib_config,
+                                    "zlib/all/conanfile.py": str(zlib),
+                                    "zlib/all/conandata.yml": conandata_yml})
+        client = TestClient()
+        client.run(f"remote add local '{folder}'")
+        client.run("install --requires=zlib/0.1@myuser/mychannel --build=missing")
+        assert "zlib/0.1@myuser/mychannel:" in client.out
+        client.run("list * -r=local")
+        assert "zlib/0.1@myuser/mychannel" in client.out
+
 
 class TestRestrictedOperations:
     def test_upload(self):