Repository: conan-io/conan
Pull request #17524 (2024-12-25 12:08:25)

Add example test for forwards compat of custom setting
Changelog: Omit
Docs: Omit

As part of the answer for https://github.com/conan-io/conan/issues/17523, I thought that a test that reflects the desired behaviour would be great to have, as a reference point for the future

Update: Now seeing the linked issue, OP was not exactly following this approach, but I think it's still worth it to add something like this


Patch:
diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py
index 065af3098..ed3cf8ed7 100644
--- a/conan/internal/model/conf.py
+++ b/conan/internal/model/conf.py
@@ -33,6 +33,7 @@ BUILT_IN_CONFS = {
     "core.sources:download_urls": "List of URLs to download backup sources from",
     "core.sources:upload_url": "Remote URL to upload backup sources to",
     "core.sources:exclude_urls": "URLs which will not be backed up",
+    "core.sources.patch:extra_path": "Extra path to search for patch files for conan create",
     # Package ID
     "core.package_id:default_unknown_mode": "By default, 'semver_mode'",
     "core.package_id:default_non_embed_mode": "By default, 'minor_mode'",
diff --git a/conan/tools/files/conandata.py b/conan/tools/files/conandata.py
index 3b1c47600..3fa41d59f 100644
--- a/conan/tools/files/conandata.py
+++ b/conan/tools/files/conandata.py
@@ -69,5 +69,8 @@ def trim_conandata(conanfile, raise_if_missing=True):
         if version_data is not None:
             result[k] = {version: version_data}
 
+    # Update the internal conanfile data too
+    conanfile.conan_data = result
+
     new_conandata_yml = yaml.safe_dump(result, default_flow_style=False)
     save(path, new_conandata_yml)
diff --git a/conan/tools/files/patches.py b/conan/tools/files/patches.py
index 5c35641d7..834a61b68 100644
--- a/conan/tools/files/patches.py
+++ b/conan/tools/files/patches.py
@@ -3,9 +3,11 @@ import os
 import shutil
 
 import patch_ng
+import yaml
 
 from conan.errors import ConanException
-from conans.util.files import mkdir
+from conan.internal.paths import DATA_YML
+from conans.util.files import mkdir, load, save
 
 
 class PatchLogHandler(logging.Handler):
@@ -122,27 +124,60 @@ def export_conandata_patches(conanfile):
     if conanfile.conan_data is None:
         raise ConanException("conandata.yml not defined")
 
-    patches = conanfile.conan_data.get('patches')
-    if patches is None:
-        conanfile.output.info("export_conandata_patches(): No patches defined in conandata")
-        return
+    conanfile_patches = conanfile.conan_data.get('patches')
 
-    if isinstance(patches, dict):
-        assert conanfile.version, "Can only be exported if conanfile.version is already defined"
-        entries = patches.get(conanfile.version, [])
-        if entries is None:
-            conanfile.output.warning(f"export_conandata_patches(): No patches defined for version {conanfile.version} in conandata.yml")
+    def _handle_patches(patches, patches_folder):
+        if patches is None:
+            conanfile.output.info("export_conandata_patches(): No patches defined in conandata")
             return
-    elif isinstance(patches, list):
-        entries = patches
-    else:
-        raise ConanException("conandata.yml 'patches' should be a list or a dict {version: list}")
-    for it in entries:
-        patch_file = it.get("patch_file")
-        if patch_file:
-            src = os.path.join(conanfile.recipe_folder, patch_file)
-            dst = os.path.join(conanfile.export_sources_folder, patch_file)
-            if not os.path.exists(src):
-                raise ConanException(f"Patch file does not exist: '{src}'")
-            mkdir(os.path.dirname(dst))
-            shutil.copy2(src, dst)
+
+        if isinstance(patches, dict):
+            assert conanfile.version, "Can only be exported if conanfile.version is already defined"
+            entries = patches.get(conanfile.version, [])
+            if entries is None:
+                conanfile.output.warning("export_conandata_patches(): No patches defined for "
+                                         f"version {conanfile.version} in conandata.yml")
+                return
+        elif isinstance(patches, list):
+            entries = patches
+        else:
+            raise ConanException("conandata.yml 'patches' should be a list or a dict "
+                                 "{version: list}")
+        for it in entries:
+            patch_file = it.get("patch_file")
+            if patch_file:
+                src = os.path.join(patches_folder, patch_file)
+                dst = os.path.join(conanfile.export_sources_folder, patch_file)
+                if not os.path.exists(src):
+                    raise ConanException(f"Patch file does not exist: '{src}'")
+                mkdir(os.path.dirname(dst))
+                shutil.copy2(src, dst)
+        return entries
+
+    _handle_patches(conanfile_patches, conanfile.recipe_folder)
+
+    extra_path = conanfile.conf.get("core.sources.patch:extra_path")
+    if extra_path:
+        if not os.path.isdir(extra_path):
+            raise ConanException(f"Patches extra path '{extra_path}' does not exist")
+        pkg_path = os.path.join(extra_path, conanfile.name)
+        if not os.path.isdir(pkg_path):
+            return
+        data_path = os.path.join(pkg_path, DATA_YML)
+        try:
+            data = yaml.safe_load(load(data_path))
+        except Exception as e:
+            raise ConanException("Invalid yml format at {}: {}".format(data_path, e))
+        data = data or {}
+        conanfile.output.info(f"Applying extra patches 'core.sources.patch:extra_path': {data_path}")
+        new_patches = _handle_patches(data.get('patches'), pkg_path)
+
+        # Update the CONANDATA.YML
+        conanfile_patches = conanfile_patches or {}
+        conanfile_patches.setdefault(conanfile.version, []).extend(new_patches)
+
+        conanfile.conan_data['patches'] = conanfile_patches
+        # Saving in the EXPORT folder
+        conanfile_data_path = os.path.join(conanfile.export_folder, DATA_YML)
+        new_conandata_yml = yaml.safe_dump(conanfile.conan_data, default_flow_style=False)
+        save(conanfile_data_path, new_conandata_yml)
diff --git a/conans/client/graph/compatibility.py b/conans/client/graph/compatibility.py
index 85e615502..d69e64eff 100644
--- a/conans/client/graph/compatibility.py
+++ b/conans/client/graph/compatibility.py
@@ -15,9 +15,40 @@ from cppstd_compat import cppstd_compat
 
 
 def compatibility(conanfile):
-    configs = cppstd_compat(conanfile)
-    # TODO: Append more configurations for your custom compatibility rules
-    return configs
+    # By default, different compiler.cppstd are compatible
+    # factors is a list of lists
+    factors = cppstd_compat(conanfile)
+
+    # MSVC 194->193 fallback compatibility
+    compiler = conanfile.settings.get_safe("compiler")
+    compiler_version = conanfile.settings.get_safe("compiler.version")
+    if compiler == "msvc":
+        msvc_fallback = {"194": "193"}.get(compiler_version)
+        if msvc_fallback:
+            factors.append([{"compiler.version": msvc_fallback}])
+
+    # Append more factors for your custom compatibility rules here
+
+    # Combine factors to compute all possible configurations
+    combinations = _factors_combinations(factors)
+    # Final compatibility settings combinations to check
+    return [{"settings": [(k, v) for k, v in comb.items()]} for comb in combinations]
+
+
+def _factors_combinations(factors):
+    combinations = []
+    for factor in factors:
+        if not combinations:
+            combinations = factor
+            continue
+        new_combinations = []
+        for comb in combinations:
+            for f in factor:
+                new_comb = comb.copy()
+                new_comb.update(f)
+                new_combinations.append(new_comb)
+        combinations.extend(new_combinations)
+    return combinations
 """
 
 
@@ -51,29 +82,7 @@ def cppstd_compat(conanfile):
             conanfile.output.warning(f'No cstd compatibility defined for compiler "{compiler}"')
         else:
             factors.append([{"compiler.cstd": v} for v in cstd_possible_values if v != cstd])
-
-    if compiler == "msvc":
-        msvc_fallback = {"194": "193"}.get(compiler_version)
-        if msvc_fallback:
-            factors.append([{"compiler.version": msvc_fallback}])
-
-    combinations = []
-    for factor in factors:
-        if not combinations:
-            combinations = factor
-            continue
-        new_combinations = []
-        for comb in combinations:
-            for f in factor:
-                new_comb = comb.copy()
-                new_comb.update(f)
-                new_combinations.append(new_comb)
-        combinations.extend(new_combinations)
-
-    ret = []
-    for comb in combinations:
-        ret.append({"settings": [(k, v) for k, v in comb.items()]})
-    return ret
+    return factors
 """
 
 


Test patch:
diff --git a/test/functional/tools/test_files.py b/test/functional/tools/test_files.py
index 44b0c2748..9abc601e4 100644
--- a/test/functional/tools/test_files.py
+++ b/test/functional/tools/test_files.py
@@ -6,8 +6,9 @@ import pytest
 
 from conan.test.assets.genconanfile import GenConanfile
 from conan.test.utils.file_server import TestFileServer
+from conan.test.utils.test_files import temp_folder
 from conan.test.utils.tools import TestClient
-from conans.util.files import save
+from conans.util.files import save, load
 
 
 class MockPatchset:
@@ -365,7 +366,7 @@ def test_relate_base_path_all_versions(mock_patch_ng):
     assert mock_patch_ng.apply_args[1:] == (0, False)
 
 
-def test_export_conandata_patches(mock_patch_ng):
+def test_export_conandata_patches():
     conanfile = textwrap.dedent("""
         import os
         from conan import ConanFile
@@ -402,7 +403,7 @@ def test_export_conandata_patches(mock_patch_ng):
     # wrong patches
     client.save({"conandata.yml": "patches: 123"})
     client.run("create .", assert_error=True)
-    assert "conandata.yml 'patches' should be a list or a dict"  in client.out
+    assert "conandata.yml 'patches' should be a list or a dict" in client.out
 
     # No patch found
     client.save({"conandata.yml": conandata_yml})
@@ -447,3 +448,60 @@ def test_export_conandata_patches_no_patches():
     client.save({"conandata.yml": conandata_yml, "conanfile.py": conanfile})
     client.run("create .")
     assert "No patches defined for version 1.0 in conandata.yml" in client.out
+
+
+@pytest.mark.parametrize("trim", [True, False])
+def test_export_conandata_patches_extra_origin(trim):
+    conanfile = textwrap.dedent(f"""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import export_conandata_patches, load, trim_conandata
+
+        class Pkg(ConanFile):
+            name = "mypkg"
+            version = "1.0"
+
+            def export(self):
+                if {trim}:
+                    trim_conandata(self)
+
+            def layout(self):
+                self.folders.source = "source_subfolder"
+
+            def export_sources(self):
+                export_conandata_patches(self)
+
+            def source(self):
+                self.output.info(load(self, os.path.join(self.export_sources_folder,
+                                                         "patches/mypatch.patch")))
+        """)
+    client = TestClient(light=True)
+    patches_folder = temp_folder()
+    conandata_yml = textwrap.dedent("""
+        patches:
+            "1.0":
+                - patch_file: "patches/mypatch.patch"
+        """)
+    save(os.path.join(patches_folder, "mypkg", "conandata.yml"), conandata_yml)
+    save(os.path.join(patches_folder, "mypkg", "patches", "mypatch.patch"), "mypatch!!!")
+
+    pkg_conandata = textwrap.dedent("""\
+        patches:
+            "1.1":
+                - patch_file: "patches/mypatch2.patch"
+    """)
+    client.save({"conanfile.py": conanfile,
+                 "conandata.yml": pkg_conandata,
+                 "patches/mypatch2.patch": ""})
+    client.run(f'create . -cc core.sources.patch:extra_path="{patches_folder}"')
+    assert "mypkg/1.0: Applying extra patches" in client.out
+    assert "mypkg/1.0: mypatch!!!" in client.out
+
+    conandata = load(client.exported_layout().conandata())
+    assert "1.0" in conandata
+    assert "patch_file: patches/mypatch.patch" in conandata
+
+    if trim:
+        assert "1.1" not in conandata
+    else:
+        assert "1.1" in conandata
diff --git a/test/integration/package_id/compatible_test.py b/test/integration/package_id/compatible_test.py
index 5d384b7df..8758d8629 100644
--- a/test/integration/package_id/compatible_test.py
+++ b/test/integration/package_id/compatible_test.py
@@ -443,13 +443,18 @@ class TestNewCompatibility:
                    compiler.runtime=dynamic
                    """)
         tc.save({"dep/conanfile.py": GenConanfile("dep", "1.0").with_setting("compiler"),
-                 "conanfile.py": GenConanfile("app", "1.0").with_require("dep/1.0").with_setting("compiler"),
                  "profile": profile})
 
         tc.run("create dep -pr=profile -s compiler.cppstd=20")
-        tc.run("create . -pr=profile -s compiler.cppstd=17")
+        tc.run("install --requires=dep/1.0 -pr=profile -s compiler.cppstd=17")
         tc.assert_listed_binary({"dep/1.0": ("b6d26a6bc439b25b434113982791edf9cab4d004", "Cache")})
 
+        tc.run("remove * -c")
+        tc.run("create dep -pr=profile -s compiler.version=193 -s compiler.cppstd=20")
+        tc.run("install --requires=dep/1.0 -pr=profile -s compiler.cppstd=17")
+        assert "compiler.cppstd=20, compiler.version=193" in tc.out
+        tc.assert_listed_binary({"dep/1.0": ("535899bb58c3ca7d80a380313d31f4729e735d1c", "Cache")})
+
 
 class TestCompatibleBuild:
     def test_build_compatible(self):
@@ -706,3 +711,58 @@ class TestCompatibleBuild:
             pkga = liba["packages"][0][pkg_index]
             assert pkga["info"]["compatibility_delta"] == {"settings": [["compiler.cppstd", "14"]]}
             assert pkga["build_args"] == "--requires=liba/0.1 --build=compatible:liba/0.1"
+
+
+def test_compatibility_new_setting_forwards_compat():
+    """ This test tries to reflect the following scenario:
+    - User adds a new setting (libc.version in this case)
+    - This setting is forward compatible
+    How is it solved with compatibility.py? Like this:
+    """
+    tc = TestClient()
+    tc.save_home({"settings_user.yml": "libc_version: [1, 2, 3]"})
+    tc.save({"conanfile.py": GenConanfile("dep", "1.0").with_settings("libc_version", "compiler")})
+    # The extra cppstd and compiler versions are for later demonstrations of combinations of settings
+    # The cppstd=17 and compiler.version=193 are used thought until the last 2 install calls
+    tc.run("create . -s=libc_version=2 -s=compiler.cppstd=17")
+    dep_package_id = tc.created_package_id("dep/1.0")
+    tc.run("install --requires=dep/1.0 -s=libc_version=3 -s=compiler.cppstd=17", assert_error=True)
+    # We can't compile, because the dep is not compatible
+    assert "Missing prebuilt package for 'dep/1.0'" in tc.out
+
+    # Let's create a compatibility extensions
+    libc_compat = textwrap.dedent("""
+        from conan.tools.scm import Version
+
+        def libc_compat(conanfile):
+            # Do we have the setting?
+            libc_version = conanfile.settings.get_safe("libc_version")
+            if libc_version is None:
+                return []
+            available_libc_versions = conanfile.settings.libc_version.possible_values()
+            ret = []
+            for possible_libc_version in available_libc_versions:
+                if Version(possible_libc_version) < Version(libc_version):
+                    ret.append({"libc_version": possible_libc_version})
+            return ret
+        """)
+    compat = tc.load_home("extensions/plugins/compatibility/compatibility.py")
+    compat = "from libc_compat import libc_compat\n" + compat
+    compat = compat.replace("# Append more factors for your custom compatibility rules here",
+                            "factors.append(libc_compat(conanfile))")
+    tc.save_home({"extensions/plugins/compatibility/libc_compat.py": libc_compat,
+                  "extensions/plugins/compatibility/compatibility.py": compat})
+
+    # Now we try again, this time app will find the compatible dep with libc_version 2
+    tc.run("install --requires=dep/1.0 -s=libc_version=3 -s=compiler.cppstd=17")
+    assert f"dep/1.0: Found compatible package '{dep_package_id}'" in tc.out
+
+    # And now we try to create the app with libc_version 1, which is still not compatible
+    tc.run("install --requires=dep/1.0 -s=libc_version=1 -s=compiler.cppstd=17", assert_error=True)
+    assert "Missing prebuilt package for 'dep/1.0'" in tc.out
+
+    # Now we try again, this time app will find the compatible dep with libc_version 2
+    # And see how we're also compatible over a different cppstd
+    tc.run("install --requires=dep/1.0 -s=libc_version=3 -s=compiler.cppstd=14")
+    assert f"dep/1.0: Found compatible package '{dep_package_id}': compiler.cppstd=17, " \
+           f"libc_version=2" in tc.out