Repository: conan-io/conan
Pull request #16073 (2024-04-12 11:54:35)

Package vendor new feature
Changelog: Feature: New ``vendor=True`` package creation and build isolation strategy
Docs: https://github.com/conan-io/docs/pull/3756

Close https://github.com/conan-io/conan/issues/13171


Patch:
diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py
index fe16deaf0..13fa6d337 100644
--- a/conan/cli/commands/create.py
+++ b/conan/cli/commands/create.py
@@ -95,6 +95,9 @@ def create(conan_api, parser, *args):
     else:
         requires = [ref] if not is_build else None
         tool_requires = [ref] if is_build else None
+        if conanfile.vendor:  # Automatically allow repackaging for conan create
+            pr = profile_build if is_build else profile_host
+            pr.conf.update("&:tools.graph:vendor", "build")
         deps_graph = conan_api.graph.load_graph_requires(requires, tool_requires,
                                                          profile_host=profile_host,
                                                          profile_build=profile_build,
diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py
index 417969850..67db93e2f 100644
--- a/conan/cli/formatters/graph/info_graph_html.py
+++ b/conan/cli/formatters/graph/info_graph_html.py
@@ -144,6 +144,11 @@ graph_info_html = r"""
                     if (node.recipe == "Platform") {
                         font.background = "Violet";
                     }
+                    if (node.vendor) {
+                        borderColor = "Red";
+                        shapeProperties = {borderDashes: [3,5]};
+                        borderWidth = 2;
+                    }
                     nodes.push({
                         id: node_id,
                         font: font,
@@ -225,6 +230,14 @@ graph_info_html = r"""
                     font: {size: 35, color: "white"},
                     color: {border: "SkyBlue", background: "Black"}
                 });
+                counter++;
+
+                legend_nodes.push({x: x + counter*step, y: y, shape: "box",
+                    label: "vendor", font: {size: 35},
+                    color: {border: "Red"},
+                    shapeProperties: {borderDashes: [3,5]},
+                    borderWidth: 2
+                });
                 return {nodes: new vis.DataSet(legend_nodes)};
             }
             let error = document.getElementById("error");
diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py
index 23b5e28f5..79ba4de0c 100644
--- a/conans/client/conf/__init__.py
+++ b/conans/client/conf/__init__.py
@@ -19,42 +19,43 @@ os:
                     "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6", "13.7",
                     "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "14.6", "14.7", "14.8",
                     "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6", "16.0", "16.1",
-                    "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1"]
+                    "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1", "17.2", "17.3", "17.4", "17.5"]
         sdk: ["iphoneos", "iphonesimulator"]
         sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4",
                         "13.0", "13.1", "13.2", "13.4", "13.5", "13.6", "13.7",
                         "14.0", "14.1", "14.2", "14.3", "14.4", "14.5", "15.0", "15.2", "15.4",
-                        "15.5", "16.0", "16.1", "16.2", "16.4", "17.0", "17.1"]
+                        "15.5", "16.0", "16.1", "16.2", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5"]
     watchOS:
         version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2",
                     "7.0", "7.1", "7.2", "7.3", "7.4", "7.5", "7.6", "8.0", "8.1", "8.3", "8.4",
                     "8.5", "8.6", "8.7", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9.6",
-                    "10.0", "10.1"]
+                    "10.0", "10.1", "10.2", "10.3", "10.4", "10.5"]
         sdk: ["watchos", "watchsimulator"]
         sdk_version: [null, "4.3", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1", "6.2",
                         "7.0", "7.1", "7.2", "7.4", "8.0", "8.0.1", "8.3", "8.5", "9.0", "9.1",
-                        "9.4", "10.0", "10.1"]
+                        "9.4", "10.0", "10.1", "10.2", "10.4", "10.5"]
     tvOS:
         version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1", "12.2", "12.3", "12.4",
                     "13.0", "13.2", "13.3", "13.4", "14.0", "14.2", "14.3", "14.4", "14.5",
                     "14.6", "14.7", "15.0", "15.1", "15.2", "15.3", "15.4", "15.5", "15.6",
-                    "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1"]
+                    "16.0", "16.1", "16.2", "16.3", "16.4", "16.5", "16.6", "17.0", "17.1", "17.2", "17.3", "17.4",
+                    "17.5"]
         sdk: ["appletvos", "appletvsimulator"]
         sdk_version: [null, "11.3", "11.4", "12.0", "12.1", "12.2", "12.4",
                         "13.0", "13.1", "13.2", "13.4", "14.0", "14.2", "14.3", "14.5", "15.0",
-                        "15.2", "15.4", "16.0", "16.1", "16.4", "17.0", "17.1"]
+                        "15.2", "15.4", "16.0", "16.1", "16.4", "17.0", "17.1", "17.2", "17.4", "17.5"]
     visionOS:
-        version: ["1.0"]
+        version: ["1.0", "1.1", "1.2"]
         sdk: ["xros", "xrsimulator"]
-        sdk_version: [null, "1.0"]
+        sdk_version: [null, "1.0", "1.1", "1.2"]
     Macos:
         version: [null, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14", "10.15",
                     "11.0", "11.1", "11.2", "11.3", "11.4", "11.5", "11.6", "11.7",
                     "12.0", "12.1", "12.2", "12.3", "12.4", "12.5", "12.6", "12.7",
                     "13.0", "13.1", "13.2", "13.3", "13.4", "13.5", "13.6",
-                    "14.0", "14.1", "14.2", "14.3", "14.4"]
+                    "14.0", "14.1", "14.2", "14.3", "14.4", "14.5"]
         sdk_version: [null, "10.13", "10.14", "10.15", "11.0", "11.1", "11.3", "12.0", "12.1",
-                        "12.3", "13.0", "13.1", "13.3", "14.0"]
+                        "12.3", "13.0", "13.1", "13.3", "14.0", "14.2", "14.4", "14.5"]
         subsystem:
             null:
             catalyst:
diff --git a/conans/client/graph/compute_pid.py b/conans/client/graph/compute_pid.py
index 60842cefa..cbfff7099 100644
--- a/conans/client/graph/compute_pid.py
+++ b/conans/client/graph/compute_pid.py
@@ -38,6 +38,9 @@ def compute_package_id(node, new_config, config_version):
             else:
                 data[require] = req_info
 
+    if conanfile.vendor:  # Make the package_id fully independent of dependencies versions
+        data, build_data = OrderedDict(), OrderedDict()  # TODO, cleaner, now minimal diff
+
     reqs_info = RequirementsInfo(data)
     build_requires_info = RequirementsInfo(build_data)
     python_requires = PythonRequiresInfo(python_requires, python_mode)
diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py
index 453cbf851..74a7f0273 100644
--- a/conans/client/graph/graph.py
+++ b/conans/client/graph/graph.py
@@ -108,6 +108,8 @@ class Node(object):
         self.transitive_deps.pop(require, None)
         self.transitive_deps[require] = TransitiveRequirement(require, node)
 
+        if self.conanfile.vendor:
+            return
         # Check if need to propagate downstream
         if not self.dependants:
             return
@@ -154,6 +156,8 @@ class Node(object):
         # Check if need to propagate downstream
         # Then propagate downstream
 
+        if self.conanfile.vendor:
+            return result
         # Seems the algrithm depth-first, would only have 1 dependant at most to propagate down
         # at any given time
         if not self.dependants:
diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py
index c343ccd2b..fc1980d99 100644
--- a/conans/client/graph/graph_binaries.py
+++ b/conans/client/graph/graph_binaries.py
@@ -181,6 +181,15 @@ class GraphBinariesAnalyzer(object):
             node.build_allowed = True
             node.binary = BINARY_BUILD if not node.cant_build else BINARY_INVALID
 
+        if node.binary == BINARY_BUILD:
+            conanfile = node.conanfile
+            if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)):
+                node.conanfile.info.invalid = f"The package '{conanfile.ref}' is a vendoring one, " \
+                                              f"needs to be built from source, but it " \
+                                              "didn't enable 'tools.graph:vendor=build' to compute " \
+                                              "its dependencies"
+                node.binary = BINARY_INVALID
+
     def _process_node(self, node, build_mode, remotes, update):
         # Check that this same reference hasn't already been checked
         if self._evaluate_is_cached(node):
diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py
index d248d25bb..41793dee9 100644
--- a/conans/client/graph/graph_builder.py
+++ b/conans/client/graph/graph_builder.py
@@ -5,7 +5,7 @@ from collections import deque
 from conan.internal.cache.conan_reference_layout import BasicLayout
 from conans.client.conanfile.configure import run_configure_method
 from conans.client.graph.graph import DepsGraph, Node, CONTEXT_HOST, \
-    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL
+    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE
 from conans.client.graph.graph import RECIPE_PLATFORM
 from conans.client.graph.graph_error import GraphLoopError, GraphConflictError, GraphMissingError, \
     GraphRuntimeError, GraphError
@@ -52,7 +52,10 @@ class DepsGraphBuilder(object):
                     continue
                 new_node = self._expand_require(require, node, dep_graph, profile_host,
                                                 profile_build, graph_lock)
-                if new_node:
+                if new_node and (not new_node.conanfile.vendor
+                                 or new_node.recipe == RECIPE_EDITABLE or
+                                 new_node.conanfile.conf.get("tools.graph:vendor",
+                                                             choices=("build",))):
                     self._initialize_requires(new_node, dep_graph, graph_lock, profile_build,
                                               profile_host)
                     open_requires.extendleft((r, new_node)
@@ -386,6 +389,7 @@ class DepsGraphBuilder(object):
     @staticmethod
     def _compute_down_options(node, require, new_ref):
         # The consumer "up_options" are the options that come from downstream to this node
+        visible = require.visible and not node.conanfile.vendor
         if require.options is not None:
             # If the consumer has specified "requires(options=xxx)", we need to use it
             # It will have less priority than downstream consumers
@@ -395,11 +399,11 @@ class DepsGraphBuilder(object):
             # options["dep"].opt=value only propagate to visible and host dependencies
             # we will evaluate if necessary a potential "build_options", but recall that it is
             # now possible to do "self.build_requires(..., options={k:v})" to specify it
-            if require.visible:
+            if visible:
                 # Only visible requirements in the host context propagate options from downstream
                 down_options.update_options(node.conanfile.up_options)
         else:
-            if require.visible:
+            if visible:
                 down_options = node.conanfile.up_options
             elif not require.build:  # for requires in "host", like test_requires, pass myoptions
                 down_options = node.conanfile.private_up_options
diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py
index 7fcdbf8ac..a174b260b 100644
--- a/conans/model/conan_file.py
+++ b/conans/model/conan_file.py
@@ -47,8 +47,8 @@ class ConanFile:
     default_options = None
     default_build_options = None
     package_type = None
+    vendor = False
     languages = []
-
     implements = []
 
     provides = None
@@ -170,6 +170,7 @@ class ConanFile:
         result["label"] = self.display_name
         if self.info is not None:
             result["info"] = self.info.serialize()
+        result["vendor"] = self.vendor
         return result
 
     @property
diff --git a/conans/model/conf.py b/conans/model/conf.py
index e11037845..49a27c3ff 100644
--- a/conans/model/conf.py
+++ b/conans/model/conf.py
@@ -83,6 +83,7 @@ BUILT_IN_CONFS = {
     "tools.files.download:retry": "Number of retries in case of failure when downloading",
     "tools.files.download:retry_wait": "Seconds to wait between download attempts",
     "tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing",
+    "tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them",
     "tools.graph:skip_binaries": "Allow the graph to skip binaries not needed in the current configuration (True by default)",
     "tools.gnu:make_program": "Indicate path to make program",
     "tools.gnu:define_libcxx11_abi": "Force definition of GLIBCXX_USE_CXX11_ABI=1 for libstdc++11",


Test patch:
diff --git a/test/integration/command/info/info_test.py b/test/integration/command/info/info_test.py
index be8b53186..e2bde2a0a 100644
--- a/test/integration/command/info/info_test.py
+++ b/test/integration/command/info/info_test.py
@@ -4,7 +4,7 @@ import textwrap
 
 from conan.cli.exit_codes import ERROR_GENERAL
 from conans.model.recipe_ref import RecipeReference
-from conan.test.utils.tools import TestClient, GenConanfile, TurboTestClient
+from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, GenConanfile, TurboTestClient
 
 
 class TestBasicCliOutput:
@@ -425,3 +425,27 @@ def test_graph_info_user():
     assert "pkg/0.1@user" in c.out
     c.run("graph info . --channel=channel")
     assert "pkg/0.1@user/channel" in c.out
+
+
+def test_graph_info_bundle():
+    c = TestClient(light=True)
+    c.save({"subfolder/conanfile.py": GenConanfile("liba", "1.0")})
+    c.run("create ./subfolder")
+    conanfile = textwrap.dedent("""
+        from conan import ConanFile
+        class RepackageRecipe(ConanFile):
+            name = "lib"
+            version = "1.0"
+            def requirements(self):
+                self.requires("liba/1.0")
+            vendor = True
+        """)
+    c.save({"conanfile.py": conanfile})
+    c.run("create .")
+    c.save({"conanfile.py": GenConanfile("consumer", "1.0").with_requires("lib/1.0")})
+
+    c.run("graph info . --build='lib*'")
+    c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Invalid")})
+
+    c.run("graph info . -c tools.graph:vendor=build --build='lib*'")
+    c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Build")})
diff --git a/test/integration/command_v2/test_inspect.py b/test/integration/command_v2/test_inspect.py
index 8de7c470a..cd7445800 100644
--- a/test/integration/command_v2/test_inspect.py
+++ b/test/integration/command_v2/test_inspect.py
@@ -22,7 +22,9 @@ def test_basic_inspect():
                      "    shared: ['True', 'False']",
                      'package_type: None',
                      'requires: []',
-                     'revision_mode: hash']
+                     'revision_mode: hash',
+                     'vendor: False'
+                     ]
 
 
 def test_options_description():
@@ -92,6 +94,7 @@ def test_normal_inspect():
                                    'package_type: None',
                                    'requires: []',
                                    'revision_mode: hash',
+                                   'vendor: False',
                                    'version: 1.0']
 
 
@@ -137,7 +140,8 @@ def test_requiremens_inspect():
             "False, 'test': False, 'force': False, 'direct': True, 'build': "
             "False, 'transitive_headers': None, 'transitive_libs': None, 'headers': "
             "True, 'package_id_mode': None, 'visible': True}]",
-            'revision_mode: hash'] == tc.out.splitlines()
+            'revision_mode: hash',
+            'vendor: False'] == tc.out.splitlines()
 
 
 def test_pythonrequires_remote():
diff --git a/test/integration/test_package_vendor.py b/test/integration/test_package_vendor.py
new file mode 100644
index 000000000..562ae4cd3
--- /dev/null
+++ b/test/integration/test_package_vendor.py
@@ -0,0 +1,140 @@
+import os
+import textwrap
+
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.tools import TestClient
+
+
+def test_package_vendor():
+    c = TestClient()
+    app = textwrap.dedent("""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import copy, save
+
+        class App(ConanFile):
+            name = "app"
+            version = "0.1"
+            package_type = "application"
+            vendor = True
+            requires = "pkga/0.1"
+            def package(self):
+                copy(self, "*", src=self.dependencies["pkga"].package_folder,
+                     dst=self.package_folder)
+                save(self, os.path.join(self.package_folder, "app.exe"), "app")
+            """)
+
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+                                                            .with_package_file("pkga.dll", "dll"),
+            "app/conanfile.py": app
+            })
+    c.run("create pkga")
+    c.run("create app")  # -c tools.graph:vendor=build will be automatic
+    assert "app/0.1: package(): Packaged 1 '.dll' file: pkga.dll" in c.out
+
+    # we can safely remove pkga
+    c.run("remove pkg* -c")
+    c.run("list app:*")
+    assert "pkga" not in c.out  # The binary doesn't depend on pkga
+    c.run("install --requires=app/0.1 --deployer=full_deploy")
+    assert "pkga" not in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # we can create a modified pkga
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+           .with_package_file("pkga.dll", "newdll")})
+    c.run("create pkga")
+    # still using the re-packaged one
+    c.run("install --requires=app/0.1 --deployer=full_deploy")
+    assert "pkga" not in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # but we can force the expansion, still not the rebuild
+    c.run("install --requires=app/0.1 --deployer=full_deploy -c tools.graph:vendor=build")
+    assert "pkga" in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # and finally we can force the expansion and the rebuild
+    c.run("install --requires=app/0.1 --build=app* --deployer=full_deploy "
+          "-c tools.graph:vendor=build")
+    assert "pkga" in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "newdll"
+    # This shoulnd't happen, no visibility over transitive dependencies of app
+    assert not os.path.exists(os.path.join(c.current_folder, "full_deploy", "host", "pkga"))
+
+    # lets remove the binary
+    c.run("remove app:* -c")
+    c.run("install --requires=app/0.1", assert_error=True)
+    assert "Missing binary" in c.out
+    c.run("install --requires=app/0.1 --build=missing", assert_error=True)
+    assert "app/0.1: Invalid: The package 'app/0.1' is a vendoring one, needs to be built " \
+           "from source, but it didn't enable 'tools.graph:vendor=build'" in c.out
+
+    c.run("install --requires=app/0.1 --build=missing  -c tools.graph:vendor=build")
+    assert "pkga" in c.out  # it works
+
+
+def test_package_vendor_editable():
+    c = TestClient()
+    pkgb = textwrap.dedent("""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import copy, save
+
+        class App(ConanFile):
+            name = "pkgb"
+            version = "0.1"
+            package_type = "shared-library"
+            vendor = True
+            requires = "pkga/0.1"
+            def layout(self):
+                self.folders.build = "build"
+                self.cpp.build.bindirs = ["build"]
+            def generate(self):
+                copy(self, "*", src=self.dependencies["pkga"].package_folder,
+                     dst=self.build_folder)
+            def build(self):
+                save(self, os.path.join(self.build_folder, "pkgb.dll"), "dll")
+            """)
+
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+                                                            .with_package_file("bin/pkga.dll", "d"),
+            "pkgb/conanfile.py": pkgb,
+            "app/conanfile.py": GenConanfile("app", "0.1").with_settings("os")
+                                                          .with_requires("pkgb/0.1")
+            })
+    c.run("create pkga")
+    c.run("editable add pkgb")
+    c.run("install app -s os=Linux")
+    assert "pkga" in c.out
+    # The environment file of "app" doesn't have any visibility of the "pkga" paths
+    envfile_app = c.load("app/conanrunenv.sh")
+    assert "pkga" not in envfile_app
+    # But the environment file needed to build "pkgb" has visibility over the "pkga" paths
+    envfile_pkgb = c.load("pkgb/conanrunenv.sh")
+    assert "pkga" in envfile_pkgb
+
+
+def test_vendor_dont_propagate_options():
+    c = TestClient()
+    app = GenConanfile("app", "0.1").with_requires("pkga/0.1").with_class_attribute("vendor=True")
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_shared_option(False),
+            "app/conanfile.py": app,
+            "consumer/conanfile.txt": "[requires]\napp/0.1",
+            "consumer_shared/conanfile.txt": "[requires]\napp/0.1\n[options]\n*:shared=True"
+            })
+    c.run("create pkga")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build")})
+    c.run("create app")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
+    c.run("install consumer --build=app/* -c tools.graph:vendor=build")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
+    c.run("install consumer_shared --build=app/* -c tools.graph:vendor=build")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})