Repository: conan-io/conan

Issue:
copy should have the option to log all files matched/copied by a call
A frequent issue we have is that copy (particularly during package) does not match the files it is intended to match, but often too many files are packaged. To allow to investigate this better, it would be good if, optionally, the files matched by each call should be logged. Currently, only the overall set of files matched by all call options, and the output is aggregated by extension if there are more than a few files.


Discussion:
Hi @sigiesec !

For now you could use `conan get` to inspect the manifest file and check what was copied to the package folder.

    conan get -p 35437353b857392790f3121d268b1db3fb9a09b0 zlib/1.2.11@conan/stable conanmanifest.txt
    1549360440
    conaninfo.txt: ef2964b8f3037c9fb22512f086cf59b7
    include/zconf.h: 8e174685774bae412b6e4aa354057f96
    include/zlib.h: 0338828e9d00c94645648b1517108324
    lib/libz.a: 668bc4404de773e98b23f8ed770dca1f
    licenses/LICENSE: 1af2e5353c16e930435fff550f827c3e
    zlib.pc: 5e60c1b132813bdcd04706c03dcb8f99

Also, you could add a hook to run `conan get` after to package.

But maybe we could customize the number of files listed when packaging by an environment variable. WDYT @memsharded ?

related issue: https://github.com/conan-io/conan/issues/4479

Regards!
Hi @uilianries, thanks for the suggestion, however this is missing a critical information, namely which call of copy caused the particular files to be copied, i.e. in particular the values of the pattern and src parameters. This is particularly hard to debug if something is wrong. 



Patch:
diff --git a/conan/tools/files/copy_pattern.py b/conan/tools/files/copy_pattern.py
index 82ee8024241..0f7f659a865 100644
--- a/conan/tools/files/copy_pattern.py
+++ b/conan/tools/files/copy_pattern.py
@@ -37,6 +37,12 @@ def copy(conanfile, pattern, src, dst, keep_path=True, excludes=None,
 
     copied_files = _copy_files(files_to_copy, src, dst, keep_path)
     copied_files.extend(_copy_files_symlinked_to_folders(files_symlinked_to_folders, src, dst))
+    if conanfile:  # Some usages still pass None
+        copied = '\n    '.join(files_to_copy)
+        conanfile.output.debug(f"copy(pattern={pattern}) copied {len(copied_files)} files\n"
+                               f"  from {src}\n"
+                               f"  to {dst}\n"
+                               f"  Files:\n    {copied}")
     return copied_files
 
 
@@ -106,10 +112,7 @@ def _copy_files(files, src, dst, keep_path):
         abs_src_name = os.path.join(src, filename)
         filename = filename if keep_path else os.path.basename(filename)
         abs_dst_name = os.path.normpath(os.path.join(dst, filename))
-        try:
-            os.makedirs(os.path.dirname(abs_dst_name))
-        except Exception:
-            pass
+        os.makedirs(os.path.dirname(abs_dst_name), exist_ok=True)
         if os.path.islink(abs_src_name):
             linkto = os.readlink(abs_src_name)  # @UndefinedVariable
             try:
diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py
index afdbd087811..4ff4f40d721 100644
--- a/conans/client/cmd/export.py
+++ b/conans/client/cmd/export.py
@@ -136,11 +136,9 @@ def export_source(conanfile, destination_source_folder):
         conanfile.exports_sources = (conanfile.exports_sources,)
 
     included_sources, excluded_sources = _classify_patterns(conanfile.exports_sources)
-    copied = []
     for pattern in included_sources:
-        _tmp = copy(conanfile, pattern, src=conanfile.recipe_folder,
-                    dst=destination_source_folder, excludes=excluded_sources)
-        copied.extend(_tmp)
+        copy(conanfile, pattern, src=conanfile.recipe_folder,
+             dst=destination_source_folder, excludes=excluded_sources)
 
     conanfile.folders.set_base_export_sources(destination_source_folder)
     _run_method(conanfile, "export_sources")
@@ -163,11 +161,9 @@ def export_recipe(conanfile, destination_folder):
 
     included_exports, excluded_exports = _classify_patterns(conanfile.exports)
 
-    copied = []
     for pattern in included_exports:
-        tmp = copy(conanfile, pattern, conanfile.recipe_folder, destination_folder,
-                   excludes=excluded_exports)
-        copied.extend(tmp)
+        copy(conanfile, pattern, conanfile.recipe_folder, destination_folder,
+             excludes=excluded_exports)
 
     conanfile.folders.set_base_export(destination_folder)
     _run_method(conanfile, "export")
diff --git a/conans/util/files.py b/conans/util/files.py
index 0a7827315c5..1005aa9b731 100644
--- a/conans/util/files.py
+++ b/conans/util/files.py
@@ -294,9 +294,9 @@ def exception_message_safe(exc):
         return repr(exc)
 
 
-def merge_directories(src, dst, excluded=None):
+def merge_directories(src, dst):
     from conan.tools.files import copy
-    copy(None, pattern="*", src=src, dst=dst, excludes=excluded)
+    copy(None, pattern="*", src=src, dst=dst)
 
 
 def gather_files(folder):


Test patch:
diff --git a/conans/test/integration/command/export_pkg_test.py b/conans/test/integration/command/export_pkg_test.py
index 37642e19423..c34f4d870f0 100644
--- a/conans/test/integration/command/export_pkg_test.py
+++ b/conans/test/integration/command/export_pkg_test.py
@@ -126,7 +126,9 @@ def package(self):
                      "libs/what": "",
                      "lib/hello.lib": "My Lib",
                      "lib/bye.txt": ""}, clean_first=True)
-        client.run("export-pkg . --user=lasote --channel=stable -s os=Windows")
+        client.run("export-pkg . --user=lasote --channel=stable -s os=Windows -vvv")
+        assert "copy(pattern=*.h) copied 1 files" in client.out
+        assert "copy(pattern=*.lib) copied 1 files" in client.out
         package_folder = client.created_layout().package()
         inc = os.path.join(package_folder, "inc")
         self.assertEqual(os.listdir(inc), ["header.h"])
diff --git a/conans/test/unittests/source/merge_directories_test.py b/conans/test/unittests/source/merge_directories_test.py
index d2763f9a594..8b1f7517420 100644
--- a/conans/test/unittests/source/merge_directories_test.py
+++ b/conans/test/unittests/source/merge_directories_test.py
@@ -5,7 +5,6 @@
 
 import pytest
 
-from conans.errors import ConanException
 from conans.test.utils.test_files import temp_folder
 from conans.util.files import mkdir, save, merge_directories, load
 
@@ -89,26 +88,3 @@ def test_parent_directory(self):
         self.assertEqual(load(join(self.dest, "file.txt")), "fromsrc")
         self.assertEqual(load(join(self.dest, "subdir2/file2.txt")), "fromdest")
         self.assertEqual(load(join(self.dest, "subdir/file2.txt")), "fromsrc")
-
-    def test_excluded_dirs(self):
-        files = ["file.txt", "subdir/file2.txt", "subdir/file3.txt", "other_dir/somefile.txt",
-                 "other_dir/somefile2.txt"]
-        self._save(self.source, files, "fromsrc")
-
-        files_dest = ["file.txt", "subdir2/file2.txt"]
-        self._save(self.dest, files_dest, "fromdest")
-
-        # Excluded one file from other_dir and the whole subdir
-        merge_directories(self.source, self.dest, excluded=["other_dir/somefile.txt", "subdir"])
-        self._assert_equals(self._get_paths(self.dest), ["file.txt",
-                                                         "subdir2/file2.txt",
-                                                         "other_dir/somefile2.txt"])
-
-        # Excluded one from dest (no sense) and one from origin
-        merge_directories(self.source, self.dest, excluded=["subdir2/file2.txt",
-                                                            "subdir",
-                                                            "other_dir/somefile.txt"])
-
-        self._assert_equals(self._get_paths(self.dest), ["file.txt",
-                                                         "subdir2/file2.txt",
-                                                         "other_dir/somefile2.txt"])