Repository: conan-io/conan
Pull request #17998 (2025-03-20 18:22:16)

[regression] `untargz` fails extracting some folders
Changelog: Bugfix: `untargz()` method was failing if directories had a more restrictive mode. 
Docs: omit
Closes: https://github.com/conan-io/conan/issues/17987

Now, it does not show the file percentage until we find a better solution for that:

**Python 3.12.3** (works as expected)

```
conanfile.py (test/1.0): Downloading 142.8MB x-tools-x86_64-bionic-linux-gnu-gcc14.tar.xz
conanfile.py (test/1.0): Unzipping x-tools-x86_64-bionic-linux-gnu-gcc14.tar.xz to /tmp/tmp52l7sgb4
```

The key part is here (from `tarfile.py`):

```python
class TarFile(object):
    # ....

    def extractall(self, path=".", members=None, *, numeric_owner=False,
                   filter=None):
        # ...
        for member in members:
            tarinfo = self._get_extract_tarinfo(member, filter_function, path)
            if tarinfo is None:
                continue
            if tarinfo.isdir():
                # For directories, delay setting attributes until later,
                # since permissions can interfere with extraction and
                # extracting contents can reset mtime.
                directories.append(tarinfo)
            self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(),
                              numeric_owner=numeric_owner)

        # Do some stuff with directories
```
In a nutshell, it's not setting the attributes for those directories at that point, it's done later. It's not the same as it's done by the `extract()` method.



Patch:
diff --git a/conan/internal/api/install/generators.py b/conan/internal/api/install/generators.py
index fe553891f..946133a7d 100644
--- a/conan/internal/api/install/generators.py
+++ b/conan/internal/api/install/generators.py
@@ -242,15 +242,20 @@ def relativize_paths(conanfile, placeholder):
     return abs_base_path, new_path
 
 
-def relativize_path(path, conanfile, placeholder):
-    abs_base_path, new_path = relativize_paths(conanfile, placeholder)
-    if abs_base_path is None:
+def relativize_path(path, conanfile, placeholder, normalize=True):
+    """
+    relative path from the "generators_folder" to "path", asuming the root file, like
+    conan_toolchain.cmake will be directly in the "generators_folder"
+    """
+    base_common_folder = conanfile.folders._base_generators # noqa
+    if not base_common_folder or not os.path.isabs(base_common_folder):
         return path
-    if path.startswith(abs_base_path):
-        path = path.replace(abs_base_path, new_path, 1)
-    else:
-        abs_base_path = abs_base_path.replace("\\", "/")
-        new_path = new_path.replace("\\", "/")
-        if path.startswith(abs_base_path):
-            path = path.replace(abs_base_path, new_path, 1)
+    try:
+        common_path = os.path.commonpath([path, conanfile.generators_folder, base_common_folder])
+        if common_path == base_common_folder:
+            rel_path = os.path.relpath(path, conanfile.generators_folder)
+            new_path = os.path.join(placeholder, rel_path)
+            return new_path.replace("\\", "/") if normalize else new_path
+    except ValueError:  # In case the unit in Windows is different, path cannot be made relative
+        pass
     return path
diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py
index 66a2a43a9..21ee2f1a0 100644
--- a/conan/tools/files/files.py
+++ b/conan/tools/files/files.py
@@ -356,15 +356,14 @@ def unzip(conanfile, filename, destination=".", keep_permissions=False, pattern=
 def untargz(filename, destination=".", pattern=None, strip_root=False, extract_filter=None):
     # NOT EXPOSED at `conan.tools.files` but used in tests
     import tarfile
-    with FileProgress(filename, msg="Uncompressing") as fileobj, tarfile.TarFile.open(fileobj=fileobj, mode='r:*') as tarredgzippedFile:
+    with tarfile.TarFile.open(filename, mode='r:*') as tarredgzippedFile:
         f = getattr(tarfile, f"{extract_filter}_filter", None) if extract_filter else None
         tarredgzippedFile.extraction_filter = f or (lambda member_, _: member_)
         if not pattern and not strip_root:
-            # Simple case: extract everything
             tarredgzippedFile.extractall(destination)
         else:
-            # Read members one by one and process them
             common_folder = None
+            members = []
             for member in tarredgzippedFile:
                 if pattern and not fnmatch(member.name, pattern):
                     continue  # Skip files that don’t match the pattern
@@ -380,7 +379,6 @@ def untargz(filename, destination=".", pattern=None, strip_root=False, extract_f
                             raise ConanException("Can't untar a tgz containing files in the root with strip_root enabled")
                     if not name.startswith(common_folder):
                         raise ConanException("The tgz file contains more than 1 folder in the root")
-
                     # Adjust the member's name for extraction
                     member.name = name[len(common_folder) + 1:]
                     member.path = member.name
@@ -388,9 +386,10 @@ def untargz(filename, destination=".", pattern=None, strip_root=False, extract_f
                         # https://github.com/conan-io/conan/issues/11065
                         member.linkpath = member.linkpath[len(common_folder) + 1:].replace("\\", "/")
                         member.linkname = member.linkpath
-                # Extract file one by one, avoiding `extractall()` with members parameter:
-                # This will avoid a first whole file read resulting in a performant improvement around 25%
-                tarredgzippedFile.extract(member, path=destination)
+                # Let's gather each member
+                members.append(member)
+            tarredgzippedFile.extractall(destination, members=members)
+
 
 def check_sha1(conanfile, file_path, signature):
     """
diff --git a/conan/tools/microsoft/msbuilddeps.py b/conan/tools/microsoft/msbuilddeps.py
index 58b0d87dd..ad2c89c84 100644
--- a/conan/tools/microsoft/msbuilddeps.py
+++ b/conan/tools/microsoft/msbuilddeps.py
@@ -185,7 +185,7 @@ class MSBuildDeps:
         root_folder = escape_path(root_folder)
         # Make the root_folder relative to the generated conan_vars_xxx.props file
         relative_root_folder = relativize_path(root_folder, self._conanfile,
-                                               "$(MSBuildThisFileDirectory)")
+                                               "$(MSBuildThisFileDirectory)", normalize=False)
 
         bin_dirs = join_paths(cpp_info.bindirs)
         res_dirs = join_paths(cpp_info.resdirs)
@@ -336,9 +336,14 @@ class MSBuildDeps:
                 public_deps = []  # To store the xml dependencies/file names
                 for required_pkg, required_comp in comp_info.parsed_requires():
                     if required_pkg is not None:  # Points to a component of a different package
-                        if required_pkg in pkg_deps:  # The transitive dep might have been skipped
-                            public_deps.append(required_pkg if required_pkg == required_comp
-                                               else "{}_{}".format(required_pkg, required_comp))
+                        try:
+                            required = pkg_deps[required_pkg]
+                        except KeyError:  # The transitive dep might have been skipped
+                            required = None
+                        if required:  # The transitive dep might have been skipped
+                            required_name = required.ref.name
+                            public_deps.append(required_name if required_pkg == required_comp
+                                               else "{}_{}".format(required_name, required_comp))
                     else:  # Points to a component of same package
                         public_deps.append("{}_{}".format(dep_name, required_comp))
                 public_deps = [self._get_valid_xml_format(d) for d in public_deps]
diff --git a/conans/client/graph/proxy.py b/conans/client/graph/proxy.py
index 66a56c283..72b927516 100644
--- a/conans/client/graph/proxy.py
+++ b/conans/client/graph/proxy.py
@@ -106,7 +106,7 @@ class ConanProxy:
                 continue
             output.info(f"Checking remote: {remote.name}")
             try:
-                if self._legacy_update and need_update:
+                if self._legacy_update and need_update and reference.revision is None:
                     if not getattr(ConanProxy, "update_policy_legacy_warning", None):
                         ConanProxy.update_policy_legacy_warning = True
                         ConanOutput().warning("The 'core:update_policy' conf is deprecated and will "


Test patch:
diff --git a/test/functional/command/test_install_deploy.py b/test/functional/command/test_install_deploy.py
index faaeb98cf..0a824b06f 100644
--- a/test/functional/command/test_install_deploy.py
+++ b/test/functional/command/test_install_deploy.py
@@ -511,9 +511,10 @@ class TestRuntimeDeployer:
         expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"])
         assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected
 
+
 @pytest.mark.parametrize("symlink, expected",
                          [(True, ["libfoo.so.0.1.0", "libfoo.so.0", "libfoo.so"]),
-                          (False, ["libfoo.so.0.1.0",])])
+                          (False, ["libfoo.so.0.1.0"])])
 def test_runtime_deploy_symlinks(symlink, expected):
     """ The deployer runtime_deploy should preserve symlinks when deploying shared libraries
     """
@@ -531,7 +532,7 @@ def test_runtime_deploy_symlinks(symlink, expected):
                        os.symlink(src="libfoo.so.0", dst="libfoo.so")
            """)
     c.save({"foo/conanfile.py": conanfile,
-            "foo/lib/libfoo.so.0.1.0": "",})
+            "foo/lib/libfoo.so.0.1.0": ""})
     c.run("export-pkg foo/ --name=foo --version=0.1.0")
     c.run(f"install --requires=foo/0.1.0 --deployer=runtime_deploy --deployer-folder=output -c:a tools.deployer:symlinks={symlink}")
 
@@ -565,3 +566,38 @@ def test_deployer_errors():
     c.run("install . --deployer=mydeploy2.py", assert_error=True)
     # The error message says conanfile, not a big deal still path to file is shown
     assert "ERROR: Unable to load conanfile" in c.out
+
+
+def test_deploy_relative_paths():
+    c = TestClient()
+    consumer = textwrap.dedent("""
+        import os
+        from conan import ConanFile
+        class Consumer(ConanFile):
+            requires = "pkg/0.1"
+            settings = "build_type"
+            os = "build_type"
+            generators = "CMakeDeps"
+            def layout(self):
+                self.folders.build = os.path.join("some/sub/folders")
+                self.folders.generators = os.path.join(self.folders.build, "generators")
+        """)
+    deploy = textwrap.dedent("""
+        import os, shutil
+        def deploy(graph, output_folder):
+            conanfile = graph.root.conanfile
+            output_folder = os.path.join(conanfile.build_folder, "installed")
+            for dep in conanfile.dependencies.values():
+                new_folder = os.path.join(output_folder, dep.ref.name)
+                shutil.copytree(dep.package_folder, new_folder, symlinks=True)
+                dep.set_deploy_folder(new_folder)
+        """)
+    c.save({"pkg/conanfile.py": GenConanfile("pkg", "0.1"),
+            "consumer/conanfile.py": consumer,
+            "mydeploy.py": deploy})
+    c.run("create pkg")
+
+    # If we don't change to another folder, the full_deploy will be recursive and fail
+    c.run("install consumer --build=missing --deployer=mydeploy.py")
+    data = c.load("consumer/some/sub/folders/generators/pkg-release-data.cmake")
+    assert 'set(pkg_PACKAGE_FOLDER_RELEASE "${CMAKE_CURRENT_LIST_DIR}/../installed/pkg")' in data
diff --git a/test/integration/command/install/install_update_test.py b/test/integration/command/install/install_update_test.py
index 450d771be..18c238f92 100644
--- a/test/integration/command/install/install_update_test.py
+++ b/test/integration/command/install/install_update_test.py
@@ -233,3 +233,15 @@ class TestUpdateOldPolicy:
         c.run("remove * -c")
         c.run("graph info --requires=pkg/0.1 --update -cc core:update_policy=legacy")
         assert f"pkg/0.1#{rev2} - Downloaded (r1)" in c.out
+
+    def test_lockfile(self):
+        # https://github.com/conan-io/conan/issues/18006
+        c = TestClient(default_server_user=True, light=True)  # needs server to fail
+        c.save_home({"global.conf": "core:update_policy=legacy"})
+        c.save({"dep/conanfile.py": GenConanfile("dep", "0.1"),
+                "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_python_requires("dep/0.1")})
+        c.run("create dep")
+        c.run("lock create pkg --lockfile-out base.lock --build=* --update")
+        c.run("lock create pkg --lockfile base.lock --lockfile-out full.lock --build=* --update")
+        # it doesn't crash
+        assert "Generated lockfile" in c.out
diff --git a/test/integration/graph/test_replace_requires.py b/test/integration/graph/test_replace_requires.py
index 4b832ebac..ef6e86bb4 100644
--- a/test/integration/graph/test_replace_requires.py
+++ b/test/integration/graph/test_replace_requires.py
@@ -418,14 +418,15 @@ class TestReplaceRequiresTransitiveGenerators:
             class App(ConanFile):
                 name = "app"
                 version = "0.1"
-                settings = "build_type"
+                settings = "build_type", "arch"
                 requires = "openssl/0.1", {zlib}
                 package_type = "application"
-                generators = "CMakeDeps", "PkgConfigDeps"
+                generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps"
             """)
         profile = textwrap.dedent("""
             [settings]
             build_type = Release
+            arch=x86_64
 
             [replace_requires]
             zlib/0.1: zlib-ng/0.1
@@ -454,6 +455,13 @@ class TestReplaceRequiresTransitiveGenerators:
         assert "set_property(TARGET openssl::openssl APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \
                '             "$<$<CONFIG:RELEASE>:ZLIB::ZLIB>")' in cmake
 
+        # checking MSBuildDeps
+        zlib_ng_props = c.load("app/conan_zlib-ng.props")
+        assert 'Project="conan_zlib-ng_release_x64.props"' in zlib_ng_props
+        props = c.load("app/conan_openssl_release_x64.props")
+        assert "<Import Condition=\"'$(conan_zlib-ng_props_imported)' != 'True'\"" \
+               " Project=\"conan_zlib-ng.props\"/>" in props
+
     @pytest.mark.parametrize("diamond", [True, False])
     def test_openssl_components(self, diamond):
         c = TestClient()
@@ -490,14 +498,15 @@ class TestReplaceRequiresTransitiveGenerators:
             class App(ConanFile):
                 name = "app"
                 version = "0.1"
-                settings = "build_type"
+                settings = "build_type", "arch"
                 requires = "openssl/0.1", {zlib}
                 package_type = "application"
-                generators = "CMakeDeps", "PkgConfigDeps"
+                generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps"
             """)
         profile = textwrap.dedent("""
             [settings]
             build_type = Release
+            arch=x86_64
 
             [replace_requires]
             zlib/0.1: zlib-ng/0.1
@@ -526,6 +535,14 @@ class TestReplaceRequiresTransitiveGenerators:
         assert "set_property(TARGET openssl::crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \
                '             "$<$<CONFIG:RELEASE>:ZLIB::ZLIB>")' in cmake
 
+        # checking MSBuildDeps
+        zlib_ng_props = c.load("app/conan_zlib-ng.props")
+        assert 'Project="conan_zlib-ng_release_x64.props"' in zlib_ng_props
+
+        props = c.load("app/conan_openssl_crypto_release_x64.props")
+        assert "<Import Condition=\"'$(conan_zlib-ng_props_imported)' != 'True'\"" \
+               " Project=\"conan_zlib-ng.props\"/>" in props
+
     @pytest.mark.parametrize("diamond", [True, False])
     @pytest.mark.parametrize("explicit_requires", [True, False])
     def test_zlib_components(self, diamond, explicit_requires):
@@ -565,14 +582,15 @@ class TestReplaceRequiresTransitiveGenerators:
             class App(ConanFile):
                 name = "app"
                 version = "0.1"
-                settings = "build_type"
+                settings = "build_type", "arch"
                 requires = "openssl/0.1", {zlib}
                 package_type = "application"
-                generators = "CMakeDeps", "PkgConfigDeps"
+                generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps"
             """)
         profile = textwrap.dedent("""
             [settings]
             build_type = Release
+            arch = x86_64
 
             [replace_requires]
             zlib/0.1: zlib-ng/0.1
@@ -604,6 +622,15 @@ class TestReplaceRequiresTransitiveGenerators:
         assert "set_property(TARGET openssl::openssl APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \
                '             "$<$<CONFIG:RELEASE>:zlib-ng::zlib-ng>")' in cmake
 
+        # checking MSBuildDeps
+        zlib_ng_props = c.load("app/conan_zlib-ng.props")
+        assert "<Import Condition=\"'$(conan_zlib-ng_myzlib_props_imported)' != 'True'\" " \
+               "Project=\"conan_zlib-ng_myzlib.props\"/" in zlib_ng_props
+
+        props = c.load("app/conan_openssl_release_x64.props")
+        assert "<Import Condition=\"'$(conan_zlib-ng_props_imported)' != 'True'\"" \
+               " Project=\"conan_zlib-ng.props\"/>" in props
+
     @pytest.mark.parametrize("diamond", [True, False])
     @pytest.mark.parametrize("package_requires", [False, True])
     def test_both_components(self, diamond, package_requires):
@@ -645,14 +672,15 @@ class TestReplaceRequiresTransitiveGenerators:
             class App(ConanFile):
                 name = "app"
                 version = "0.1"
-                settings = "build_type"
+                settings = "build_type", "arch"
                 requires = "openssl/0.1", {zlib}
                 package_type = "application"
-                generators = "CMakeDeps", "PkgConfigDeps"
+                generators = "CMakeDeps", "PkgConfigDeps", "MSBuildDeps"
             """)
         profile = textwrap.dedent("""
             [settings]
             build_type = Release
+            arch = x86_64
 
             [replace_requires]
             zlib/0.1: zlib-ng/0.1
@@ -687,3 +715,16 @@ class TestReplaceRequiresTransitiveGenerators:
         else:
             assert "set_property(TARGET openssl::crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES\n" \
                    '             "$<$<CONFIG:RELEASE>:ZLIB::ZLIB>")' in cmake
+
+        # checking MSBuildDeps
+        zlib_ng_props = c.load("app/conan_zlib-ng.props")
+        assert "<Import Condition=\"'$(conan_zlib-ng_myzlib_props_imported)' != 'True'\" " \
+               "Project=\"conan_zlib-ng_myzlib.props\"/" in zlib_ng_props
+
+        props = c.load("app/conan_openssl_crypto_release_x64.props")
+        if package_requires:
+            assert "<Import Condition=\"'$(conan_zlib-ng_props_imported)' != 'True'\"" \
+                   " Project=\"conan_zlib-ng.props\"/>" in props
+        else:
+            assert "<Import Condition=\"'$(conan_zlib-ng_myzlib_props_imported)' != 'True'\"" \
+                   " Project=\"conan_zlib-ng_myzlib.props\"/>" in props
diff --git a/test/unittests/util/files/strip_root_extract_test.py b/test/unittests/util/files/strip_root_extract_test.py
index d65bcb367..a3b9910da 100644
--- a/test/unittests/util/files/strip_root_extract_test.py
+++ b/test/unittests/util/files/strip_root_extract_test.py
@@ -272,3 +272,44 @@ class TarExtractPlainTest(unittest.TestCase):
         extract_folder = temp_folder()
         with self.assertRaisesRegex(ConanException, "Can't untar a tgz containing files in the root with strip_root enabled"):
             unzip(ConanFileMock(), tgz_file, destination=extract_folder, strip_root=True)
+
+
+def _compress_root_folder(folder, tgz_path, root_folder_name="root"):
+    """
+    Compress the whole folder adding withing the tar file also the folders. For instance:
+
+    $ tar -tf '/tmp/folder/file.tar.gz'
+    root/
+    root/parent/
+    root/parent/bin/
+    root/parent/bin/file1
+    root/parent/bin/file2
+    """
+    with open(tgz_path, "wb") as tgz_handle:
+        tgz = gzopen_without_timestamps("name", mode="w", fileobj=tgz_handle)
+        files, _ = gather_files(folder)
+        tgz.add(root_folder_name, arcname=os.path.basename(root_folder_name))
+        tgz.close()
+
+
+def test_decompressing_folders_with_different_modes():
+    """
+    When compressed folders come with restrictive permissions, untar() function should not
+    raise a PermissionError.
+
+    Issue related: https://github.com/conan-io/conan/issues/17987
+    """
+    tmp_folder = temp_folder()
+    tgz_folder = temp_folder()
+    tgz_file = os.path.join(tgz_folder, "file.tar.gz")
+    with chdir(tmp_folder):
+        save("root/parent/bin/file1", "contentsfile1")
+        save("root/parent/bin/file2", "contentsfile2")
+        os.chmod(os.path.join("root", "parent"), mode=0o555)
+        os.chmod(os.path.join("root", "parent", "bin"), mode=0o700)
+        _compress_root_folder(tmp_folder, tgz_file, root_folder_name="root")
+
+    # Tgz unzipped regularly
+    extract_folder = temp_folder()
+    # Do not raise any PermissionError
+    untargz(tgz_file, destination=extract_folder, strip_root=True)