Repository: conan-io/conan
Pull request #3065 (2018-06-17 17:37:40)

Add filter for extract tools
- [x] Refer to the issue that supports this Pull Request.
- [x] If the issue has missing info, explain the purpose/use case/pain/need that covers this Pull Request.
- [x] I've read the [Contributing guide](https://raw.githubusercontent.com/conan-io/conan/develop/.github/CONTRIBUTING.md).
- [x] I've followed the PEP8 style guides for Python code.
- [x] I've opened another PR in the Conan docs repo to the ``develop`` branch, documenting this one. Also adding a description of the changes in the ``changelog.rst`` file. https://github.com/conan-io/docs

This pull request implements #2938 
The pull request for documentation is conan-io/docs#689

CLAassistant: [![CLA assistant check](https://cla-assistant.io/pull/badge/not_signed)](https://cla-assistant.io/conan-io/conan?pullRequest=3065) <br/>Thank you for your submission, we really appreciate it. Like many open source projects, we ask that you sign our [Contributor License Agreement](https://cla-assistant.io/conan-io/conan?pullRequest=3065) before we can accept your contribution.<br/><sub>You have signed the CLA already but the status is still pending? Let us [recheck](https://cla-assistant.io/check/conan-io/conan?pullRequest=3065) it.</sub>
memsharded: CI looks good, it should have finished already, but the Windows machines seem to be frozen. We will have a look when possible, don't worry about that.
lasote: It looks like the test is failing in Windows. Thanks for your contribution!!!

Patch:
diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py
index e24f66d2a..b4de955a9 100644
--- a/conans/client/tools/files.py
+++ b/conans/client/tools/files.py
@@ -4,6 +4,7 @@ import os
 import sys
 
 from contextlib import contextmanager
+from fnmatch import fnmatch
 from patch import fromfile, fromstring
 
 from conans.client.output import ConanOutput
@@ -50,20 +51,23 @@ def human_size(size_bytes):
     return "%s%s" % (formatted_size, suffix)
 
 
-def unzip(filename, destination=".", keep_permissions=False):
+def unzip(filename, destination=".", keep_permissions=False, pattern=None):
     """
     Unzip a zipped file
     :param filename: Path to the zip file
     :param destination: Destination folder
-    :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX
-    system, the bits could produce undefined permission schema. Use only this option if you are sure that the
-    zip was created correctly.
+    :param keep_permissions: Keep the zip permissions. WARNING: Can be
+    dangerous if the zip was not created in a NIX system, the bits could
+    produce undefined permission schema. Use this option only if you are sure
+    that the zip was created correctly.
+    :param pattern: Extract only paths matching the pattern. This should be a
+    Unix shell-style wildcard, see fnmatch documentation for more details.
     :return:
     """
     if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or
             filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or
             filename.endswith(".tar")):
-        return untargz(filename, destination)
+        return untargz(filename, destination, pattern)
     import zipfile
     full_path = os.path.normpath(os.path.join(get_cwd(), destination))
 
@@ -79,7 +83,12 @@ def unzip(filename, destination=".", keep_permissions=False):
             pass
 
     with zipfile.ZipFile(filename, "r") as z:
-        uncompress_size = sum((file_.file_size for file_ in z.infolist()))
+        if not pattern:
+            zip_info = z.infolist()
+        else:
+            zip_info = list(filter(lambda zi: fnmatch(zi.filename, pattern),
+                                   z.infolist()))
+        uncompress_size = sum((file_.file_size for file_ in zip_info))
         if uncompress_size > 100000:
             _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size))
         else:
@@ -88,7 +97,7 @@ def unzip(filename, destination=".", keep_permissions=False):
 
         print_progress.last_size = -1
         if platform.system() == "Windows":
-            for file_ in z.infolist():
+            for file_ in zip_info:
                 extracted_size += file_.file_size
                 print_progress(extracted_size, uncompress_size)
                 try:
@@ -96,7 +105,7 @@ def unzip(filename, destination=".", keep_permissions=False):
                 except Exception as e:
                     _global_output.error("Error extract %s\n%s" % (file_.filename, str(e)))
         else:  # duplicated for, to avoid a platform check for each zipped file
-            for file_ in z.infolist():
+            for file_ in zip_info:
                 extracted_size += file_.file_size
                 print_progress(extracted_size, uncompress_size)
                 try:
@@ -110,10 +119,15 @@ def unzip(filename, destination=".", keep_permissions=False):
                     _global_output.error("Error extract %s\n%s" % (file_.filename, str(e)))
 
 
-def untargz(filename, destination="."):
+def untargz(filename, destination=".", pattern=None):
     import tarfile
     with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile:
-        tarredgzippedFile.extractall(destination)
+        if not pattern:
+            tarredgzippedFile.extractall(destination)
+        else:
+            members = list(filter(lambda m: fnmatch(m.name, pattern),
+                                  tarredgzippedFile.getmembers()))
+            tarredgzippedFile.extractall(destination, members=members)
 
 
 def check_with_algorithm_sum(algorithm_name, file_path, signature):
diff --git a/conans/model/settings.py b/conans/model/settings.py
index 782c94fb8..7c4441144 100644
--- a/conans/model/settings.py
+++ b/conans/model/settings.py
@@ -345,6 +345,8 @@ class Settings(object):
             other_field_def = constraint_def[field]
             if other_field_def is None:  # Means leave it as is
                 continue
+            if isinstance(other_field_def, str):
+                other_field_def = [other_field_def]
 
             values_to_remove = []
             for value in config_item.values_range:  # value = "Visual Studio"


Test patch:
diff --git a/conans/test/model/other_settings_test.py b/conans/test/model/other_settings_test.py
index 8b6d8e360..0dbba5d7d 100644
--- a/conans/test/model/other_settings_test.py
+++ b/conans/test/model/other_settings_test.py
@@ -131,6 +131,19 @@ compiler:
         self.assertIn("build_type", load(client.paths.settings_path))
         self.assertIn("390146894f59dda18c902ee25e649ef590140732", client.user_io.out)
 
+    def settings_constraint_error_type_test(self):
+        # https://github.com/conan-io/conan/issues/3022
+        conanfile = """from conans import ConanFile
+class Test(ConanFile):
+    settings = {"os": "Linux"}
+    def build(self):
+        self.output.info("OS!!: %s" % self.settings.os)
+    """
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.run("create . Pkg/0.1@user/testing -s os=Linux")
+        self.assertIn("Pkg/0.1@user/testing: OS!!: Linux", client.out)
+
     def settings_constraint_test(self):
         conanfile = """from conans import ConanFile
 class Test(ConanFile):
diff --git a/conans/test/util/files_extract_wildcard_test.py b/conans/test/util/files_extract_wildcard_test.py
new file mode 100644
index 000000000..80e99a61c
--- /dev/null
+++ b/conans/test/util/files_extract_wildcard_test.py
@@ -0,0 +1,79 @@
+import os
+import tarfile
+import zipfile
+
+from unittest import TestCase
+
+from conans.test.utils.test_files import temp_folder
+from conans.tools import unzip
+from conans.util.files import save_files
+
+
+def create_archive(archive, root, relative_file_paths):
+    """ Create an archive with given file paths relative to given root."""
+    extension = os.path.basename(archive).split(os.extsep, 1)[1]
+    if extension == "zip":
+        with zipfile.ZipFile(archive, mode="w") as z:
+            for file_ in relative_file_paths:
+                z.write(os.path.join(root, file_), arcname=file_)
+    else:
+        # Treat any other extension except "tar.gz" as a normal "tar"
+        mode = "gz" if extension == "tar.gz" else ""
+        with tarfile.open(archive, mode="w:%s" % mode) as t:
+            for file_ in relative_file_paths:
+                t.add(os.path.join(root, file_), arcname=file_)
+    return archive
+
+
+class FilesExtractPatternTest(TestCase):
+
+    def test_patterns(self):
+        def cross_platform_path(relative_path_with_slashes):
+            """ Convert string with slashes to platform-specific path. Windows
+            will not interpret correctly the slash. """
+            return os.path.join(*relative_path_with_slashes.split("/"))
+
+        # Test setup
+        src_dir = temp_folder()
+        files = {cross_platform_path(k): v for (k, v) in {
+            "foo/file.cpp": "code",
+            "foo/bar/file.txt": "text",
+            "foo/bar/file.cpp": "more code",
+            "foo/bar/baz/file.txt": "more text"
+        }.items()}
+        matches = {k: list(map(cross_platform_path, v)) for (k, v) in {
+            "*.cpp": ["foo/file.cpp",
+                      "foo/bar/file.cpp"],
+            "*.txt": ["foo/bar/file.txt",
+                      "foo/bar/baz/file.txt"],
+            "foo/bar/*": ["foo/bar/file.txt",
+                          "foo/bar/file.cpp",
+                          "foo/bar/baz/file.txt"],
+            "foo/bar/baz/*": ["foo/bar/baz/file.txt"],
+            "*": ["foo/file.cpp",
+                  "foo/bar/file.txt",
+                  "foo/bar/file.cpp",
+                  "foo/bar/baz/file.txt"],
+            "nothing": []
+        }.items()}
+        save_files(src_dir, files)
+
+        for extension in ["zip", "tar", "tar.gz"]:
+            # GIVEN an archive with files inside
+            archive_dir = temp_folder()
+            archive = os.path.join(archive_dir, "archive.%s" % extension)
+            create_archive(archive, src_dir, files)
+
+            for (pattern, paths) in matches.items():
+                # WHEN a pattern is used for file extraction
+                dst_dir = temp_folder()
+                unzip(archive, dst_dir, pattern=pattern)
+
+                # THEN only and all files matching the pattern are extracted
+                expected = set(map(lambda x: os.path.join(dst_dir, x), paths))
+                actual = set()
+                for extracted_dir, _, extracted_files in os.walk(dst_dir):
+                    actual.update(map(lambda x: os.path.join(extracted_dir, x),
+                                      extracted_files))
+
+                self.assertSetEqual(expected, actual)