Repository: conda/conda-build

Issue:
Negative matching 'files' for outputs
<!--
Thanks for opening an issue! A few things to keep in mind:

- This issue tracker is for issues with or enhancements for conda-build.

Anaconda Community Code of Conduct: https://www.anaconda.com/community-code-of-conduct/

For conda issues, you should open an issue at https://github.com/conda/conda/issues
For Anaconda issues, you should open an issue at https://github.com/ContinuumIO/anaconda-issues/issues
For an issue with a particular conda-forge package, you should open an issue on the 
corresponding feedstock: https://github.com/conda-forge
-->

### Actual Behavior

There is currently no good way to exclude one or more files from being included in an output package that uses `files`.

### Expected Behavior

Support for some sort of negative match condition that is collected and applied _after_ the positive glob expressions are resolved. Anything analogous to the `%exclude` macro you can use in RPM spec files would be great.

Perhaps something as simple as a leading exclamation `!` would work:

```yaml
outputs:
  - name: mpyackage
    files:
      - bin/mypackage*
      - "!bin/mypackage-optional*"

  - name: mypackage-optional
    files:
      - bin/mypackage-optional
```

[AFAIK the exclamation is special in YAML, so it would have to be quoted always]

I'm happy to contribute to development, if other people think this is a good idea.
Negative matching 'files' for outputs
<!--
Thanks for opening an issue! A few things to keep in mind:

- This issue tracker is for issues with or enhancements for conda-build.

Anaconda Community Code of Conduct: https://www.anaconda.com/community-code-of-conduct/

For conda issues, you should open an issue at https://github.com/conda/conda/issues
For Anaconda issues, you should open an issue at https://github.com/ContinuumIO/anaconda-issues/issues
For an issue with a particular conda-forge package, you should open an issue on the 
corresponding feedstock: https://github.com/conda-forge
-->

### Actual Behavior

There is currently no good way to exclude one or more files from being included in an output package that uses `files`.

### Expected Behavior

Support for some sort of negative match condition that is collected and applied _after_ the positive glob expressions are resolved. Anything analogous to the `%exclude` macro you can use in RPM spec files would be great.

Perhaps something as simple as a leading exclamation `!` would work:

```yaml
outputs:
  - name: mpyackage
    files:
      - bin/mypackage*
      - "!bin/mypackage-optional*"

  - name: mypackage-optional
    files:
      - bin/mypackage-optional
```

[AFAIK the exclamation is special in YAML, so it would have to be quoted always]

I'm happy to contribute to development, if other people think this is a good idea.


Discussion:
Sounds good
Sounds good

Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -1692,7 +1692,14 @@ def post_process_files(m: MetaData, initial_prefix_files):
     return new_files
 
 
-def bundle_conda(output, metadata: MetaData, env, stats, **kw):
+def bundle_conda(
+    output,
+    metadata: MetaData,
+    env,
+    stats,
+    new_prefix_files: set[str] = set(),
+    **kw,
+):
     log = utils.get_logger(__name__)
     log.info("Packaging %s", metadata.dist())
     get_all_replacements(metadata.config)
@@ -1802,10 +1809,26 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     if files:
         # Files is specified by the output
         # we exclude the list of files that we want to keep, so post-process picks them up as "new"
-        keep_files = {
-            os.path.normpath(pth)
-            for pth in utils.expand_globs(files, metadata.config.host_prefix)
-        }
+        if isinstance(files, dict):
+            # When file matching with include/exclude lists, only
+            # new_prefix_files are considered. Files in the PREFIX from other
+            # recipes (dependencies) are ignored
+            include = files.get("include") or []
+            exclude = files.get("exclude") or []
+            exclude_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(exclude, metadata.config.host_prefix)
+            }
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(include, metadata.config.host_prefix)
+            }
+            keep_files = new_prefix_files.intersection(keep_files) - exclude_files
+        else:
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(files, metadata.config.host_prefix)
+            }
         pfx_files = set(utils.prefix_files(metadata.config.host_prefix))
         initial_files = {
             item
@@ -1980,7 +2003,13 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     return final_outputs
 
 
-def bundle_wheel(output, metadata: MetaData, env, stats):
+def bundle_wheel(
+    output,
+    metadata: MetaData,
+    env,
+    stats,
+    new_prefix_files: set[str] = set(),
+):
     ext = ".bat" if utils.on_win else ".sh"
     with TemporaryDirectory() as tmpdir, utils.tmp_chdir(metadata.config.work_dir):
         dest_file = os.path.join(metadata.config.work_dir, "wheel_output" + ext)
@@ -2706,8 +2735,8 @@ def build(
                     # This is wrong, files has not been expanded at this time and could contain
                     # wildcards.  Also well, I just do not understand this, because when this
                     # does contain wildcards, the files in to_remove will slip back in.
-                    if "files" in output_d:
-                        output_d["files"] = set(output_d["files"]) - to_remove
+                    if (files := output_d.get("files")) and not isinstance(files, dict):
+                        output_d["files"] = set(files) - to_remove
 
                     # copies the backed-up new prefix files into the newly created host env
                     for f in new_prefix_files:
@@ -2722,7 +2751,9 @@ def build(
                     with utils.path_prepended(m.config.build_prefix):
                         env = environ.get_dict(m=m)
                     pkg_type = "conda" if not hasattr(m, "type") else m.type
-                    newly_built_packages = bundlers[pkg_type](output_d, m, env, stats)
+                    newly_built_packages = bundlers[pkg_type](
+                        output_d, m, env, stats, new_prefix_files
+                    )
                     # warn about overlapping files.
                     if "checksums" in output_d:
                         for file, csum in output_d["checksums"].items():


Test patch:
diff --git a/tests/test-recipes/split-packages/copying_files/bld.bat b/tests/test-recipes/split-packages/copying_files/bld.bat
--- a/tests/test-recipes/split-packages/copying_files/bld.bat
+++ b/tests/test-recipes/split-packages/copying_files/bld.bat
@@ -2,4 +2,14 @@ echo "weee" > %PREFIX%\subpackage_file1
 mkdir %PREFIX%\somedir
 echo "weee" > %PREFIX%\somedir\subpackage_file1
 echo "weee" > %PREFIX%\subpackage_file1.ext
-echo "weee" > %PREFIX%\subpackage_file2.ext
\ No newline at end of file
+echo "weee" > %PREFIX%\subpackage_file2.ext
+echo "weee" > %PREFIX%\subpackage_file3.ext
+
+echo "weee" > %PREFIX%\subpackage_include_exclude1
+mkdir %PREFIX%\anotherdir
+echo "weee" > %PREFIX%\anotherdir\subpackage_include_exclude1
+echo "weee" > %PREFIX%\subpackage_include_exclude1.wav
+echo "weee" > %PREFIX%\subpackage_include_exclude2.wav
+echo "weee" > %PREFIX%\subpackage_include_exclude3.wav
+mkdir %PREFIX%\Library\bin
+echo "weee" > %PREFIX%\Library\bin\dav1d.fake
diff --git a/tests/test-recipes/split-packages/copying_files/build.sh b/tests/test-recipes/split-packages/copying_files/build.sh
--- a/tests/test-recipes/split-packages/copying_files/build.sh
+++ b/tests/test-recipes/split-packages/copying_files/build.sh
@@ -6,3 +6,18 @@ echo "weee" > $PREFIX/somedir/subpackage_file1
 # test glob patterns
 echo "weee" > $PREFIX/subpackage_file1.ext
 echo "weee" > $PREFIX/subpackage_file2.ext
+echo "weee" > $PREFIX/subpackage_file3.ext
+
+# The files used to test the two subpackages must be disjoint because they are
+# coinstalled
+# test copying filename
+echo "weee" > $PREFIX/subpackage_include_exclude1
+# test copying by folder name
+mkdir $PREFIX/anotherdir
+echo "weee" > $PREFIX/anotherdir/subpackage_include_exclude1
+# test glob patterns
+echo "weee" > $PREFIX/subpackage_include_exclude1.wav
+echo "weee" > $PREFIX/subpackage_include_exclude2.wav
+echo "weee" > $PREFIX/subpackage_include_exclude3.wav
+mkdir $PREFIX/lib
+echo "weee" > $PREFIX/lib/libdav1d.fake
diff --git a/tests/test-recipes/split-packages/copying_files/meta.yaml b/tests/test-recipes/split-packages/copying_files/meta.yaml
--- a/tests/test-recipes/split-packages/copying_files/meta.yaml
+++ b/tests/test-recipes/split-packages/copying_files/meta.yaml
@@ -4,14 +4,44 @@ package:
 
 requirements:
   run:
-    - my_script_subpackage
+    - my_script_subpackage_files
+    - my_script_subpackage_include_exclude
 
 outputs:
-  - name: my_script_subpackage
+  - name: my_script_subpackage_files
+    build:
+      ignore_run_exports_from:
+        - libpng
+    requirements:
+      host:
+        - libpng=1.6.39
     files:
       - subpackage_file1
       - somedir
       - "*.ext"
+      # Libs should match because they are in the prefix
+      - "lib/libpng*"       # [unix]
+      - "Library/bin/libpng*"  # [win]
+    test:
+      script: subpackage_test.py
+      script_interpreter: python
+  - name: my_script_subpackage_include_exclude
+    build:
+      ignore_run_exports_from:
+        - dav1d
+    requirements:
+      host:
+        - dav1d=1.2.1
+    files:
+      include:
+        - subpackage_include_exclude1
+        - anotherdir
+        - "*.wav"
+        # Libs should not match because they come from a different package
+        - "lib/libdav1d*"       # [unix]
+        - "Library/bin/dav1d*"  # [win]
+      exclude:
+        - "*3.wav"
     test:
       script: subpackage_test.py
       script_interpreter: python
diff --git a/tests/test-recipes/split-packages/copying_files/subpackage_test.py b/tests/test-recipes/split-packages/copying_files/subpackage_test.py
--- a/tests/test-recipes/split-packages/copying_files/subpackage_test.py
+++ b/tests/test-recipes/split-packages/copying_files/subpackage_test.py
@@ -1,33 +1,76 @@
 import os
+import sys
 
-print(os.getenv('PREFIX'))
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1')
-assert os.path.isfile(filename)
+if os.getenv("PKG_NAME") == "my_script_subpackage_files":
+    file_basename = "subpackage_file"
+    dirname = "somedir"
+    extension = "ext"
+
+    if "darwin" in sys.platform:
+        external_host_file = "lib/libpng16.dylib"
+    elif "win32" in sys.platform:
+        external_host_file = "Library/bin/libpng16.dll"
+    else:
+        external_host_file = "lib/libpng16.so"
+
+    filename = os.path.join(os.environ["PREFIX"], f"{file_basename}3.{extension}")
+    print(filename)
+    assert os.path.isfile(filename), filename + " is missing"
+    print("glob files OK")
+
+    filename = os.path.join(os.environ["PREFIX"], external_host_file)
+    print(filename)
+    assert os.path.isfile(filename), filename + " is missing"
+    print("glob files prefix OK")
+
+if os.getenv("PKG_NAME") == "my_script_subpackage_include_exclude":
+    file_basename = "subpackage_include_exclude"
+    dirname = "anotherdir"
+    extension = "wav"
+
+    if "darwin" in sys.platform:
+        external_host_file = "lib/libdav1d.6.dylib"
+    elif "win32" in sys.platform:
+        external_host_file = "Library/bin/dav1d.dll"
+    else:
+        external_host_file = "lib/libdav1d.so.6"
+
+    filename = os.path.join(os.environ["PREFIX"], f"{file_basename}3.{extension}")
+    assert not os.path.isfile(filename), filename + " is missing"
+    print("glob exclude OK")
+
+    filename = os.path.join(os.environ["PREFIX"], external_host_file)
+    assert not os.path.isfile(filename), filename + " is missing"
+    print("glob exclude prefix OK")
+
+print(os.getenv("PREFIX"))
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}1")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("plain file OK")
 
-filename = os.path.join(os.environ['PREFIX'], 'somedir', 'subpackage_file1')
+filename = os.path.join(os.environ["PREFIX"], dirname, f"{file_basename}1")
 assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("subfolder file OK")
 
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file1.ext')
-assert os.path.isfile(filename)
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}1.{extension}")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 
-filename = os.path.join(os.environ['PREFIX'], 'subpackage_file2.ext')
-assert os.path.isfile(filename)
+filename = os.path.join(os.environ["PREFIX"], f"{file_basename}2.{extension}")
+assert os.path.isfile(filename), filename + " is missing"
 contents = open(filename).read().rstrip()
-if hasattr(contents, 'decode'):
+if hasattr(contents, "decode"):
     contents = contents.decode()
-assert "weee" in contents, 'incorrect file contents: %s' % contents
+assert "weee" in contents, "incorrect file contents: %s" % contents
 print("glob OK")