Repository: pypa/setuptools

Issue:
[BUG] Package data missing with Setuptools 61
I've been fighting building [one of my projects](/jaraco/recapturedocs) this evening, finding that all of the package data is missing.

```
$ pip-run -q recapturedocs==5.0.1 -- -c "import recapturedocs, os; assert 'prod.conf' in os.listdir(os.path.dirname(recapturedocs.__file__))"
Traceback (most recent call last):
  File "<string>", line 1, in <module>
AssertionError
```

But if I instead build with `setuptools<61` (by changing the pyproject.toml), the content is present.

```
$ git -C ~/m/recapturedocs diff
diff --git a/pyproject.toml b/pyproject.toml
index 190b355..cfacc1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=56", "setuptools_scm[toml]>=3.4.1"]
+requires = ["setuptools>=56,<61", "setuptools_scm[toml]>=3.4.1"]
 build-backend = "setuptools.build_meta"
 
 [tool.black]
$ pip-run -q ~/m/recapturedocs -- -c "import recapturedocs, os; assert 'prod.conf' in os.listdir(os.path.dirname(recapturedocs.__file__))"
$
```

I haven't yet had a chance to investigate why, but I wanted to register this issue.


Discussion:
I have the same issue with a package and also use `setuptools_scm` to manage the package data. If I were to speculate, something has broken with the update to `setuptools` and the current version of `setuptools_scm`.

Having said that, I see the same issue with https://github.com/MAVENSDC/cdflib and that does not use `setuptools_scm`.
I'm having a similar issue (presumably the same cause). I use `MANIFEST.in` to control which files are included in the sdist and wheel, but `MANIFEST.in` now seems to be ignored for the wheel.

Using `git bisect` I found that the behaviour changed after https://github.com/pypa/setuptools/commit/755e053ee6c3b82024a687082a164e1c4a88cfbb; if I install the previous commit (83d11a14) everything works as expected.
Specifically, commenting out these two lines makes it work for me, even on `main`:
https://github.com/pypa/setuptools/blob/4923d1139be9366b3010a6d6e56d4c4980a0ab6e/setuptools/config/setupcfg.py#L156-L157
Not sure why or what purpose those lines serve.
Hi Jason, thank you very much for reporting this. I will work on it.
Thank you very much @domdfcoding for investigating! That is very helpful.

These lines serve to ensure people can use `version = attr: module.__version__` with auto-discovery, I was not expecting them to have this side effect. I will investigate more in depth.

Resolving patch:
diff --git a/setuptools/config/expand.py b/setuptools/config/expand.py
--- a/setuptools/config/expand.py
+++ b/setuptools/config/expand.py
@@ -292,8 +292,8 @@ def find_packages(
 
     :rtype: list
     """
-
-    from setuptools.discovery import remove_nested_packages
+    from setuptools.discovery import construct_package_dir
+    from setuptools.extern.more_itertools import unique_everseen, always_iterable
 
     if namespaces:
         from setuptools.discovery import PEP420PackageFinder as PackageFinder
@@ -302,18 +302,18 @@ def find_packages(
 
     root_dir = root_dir or os.curdir
     where = kwargs.pop('where', ['.'])
-    if isinstance(where, str):
-        where = [where]
-
-    packages = []
+    packages: List[str] = []
     fill_package_dir = {} if fill_package_dir is None else fill_package_dir
-    for path in where:
-        pkgs = PackageFinder.find(_nest_path(root_dir, path), **kwargs)
+
+    for path in unique_everseen(always_iterable(where)):
+        package_path = _nest_path(root_dir, path)
+        pkgs = PackageFinder.find(package_path, **kwargs)
         packages.extend(pkgs)
-        if fill_package_dir.get("") != path:
-            parent_pkgs = remove_nested_packages(pkgs)
-            parent = {pkg: "/".join([path, *pkg.split(".")]) for pkg in parent_pkgs}
-            fill_package_dir.update(parent)
+        if pkgs and not (
+            fill_package_dir.get("") == path
+            or os.path.samefile(package_path, root_dir)
+        ):
+            fill_package_dir.update(construct_package_dir(pkgs, path))
 
     return packages
 
diff --git a/setuptools/discovery.py b/setuptools/discovery.py
--- a/setuptools/discovery.py
+++ b/setuptools/discovery.py
@@ -41,6 +41,7 @@
 import os
 from fnmatch import fnmatchcase
 from glob import glob
+from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union
 
@@ -577,3 +578,9 @@ def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -
 
     parent = package_dir.get("") or ""
     return os.path.join(root_dir, *parent.split("/"), *parts)
+
+
+def construct_package_dir(packages: List[str], package_path: _Path) -> Dict[str, str]:
+    parent_pkgs = remove_nested_packages(packages)
+    prefix = Path(package_path).parts
+    return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}


Test patch:
diff --git a/setuptools/tests/test_config_discovery.py b/setuptools/tests/test_config_discovery.py
--- a/setuptools/tests/test_config_discovery.py
+++ b/setuptools/tests/test_config_discovery.py
@@ -12,6 +12,7 @@
 import distutils.core
 
 import pytest
+import jaraco.path
 from path import Path as _Path
 
 from .contexts import quiet
@@ -398,6 +399,101 @@ def test_dont_skip_discovery_with_pyproject_metadata(self, tmp_path):
             _get_dist(tmp_path, {})
 
 
+class TestWithPackageData:
+    def _simulate_package_with_data_files(self, tmp_path, src_root):
+        files = [
+            f"{src_root}/proj/__init__.py",
+            f"{src_root}/proj/file1.txt",
+            f"{src_root}/proj/nested/file2.txt",
+        ]
+        _populate_project_dir(tmp_path, files, {})
+
+        manifest = """
+            global-include *.py *.txt
+        """
+        (tmp_path / "MANIFEST.in").write_text(DALS(manifest))
+
+    EXAMPLE_SETUPCFG = """
+    [metadata]
+    name = proj
+    version = 42
+
+    [options]
+    include_package_data = True
+    """
+    EXAMPLE_PYPROJECT = """
+    [project]
+    name = "proj"
+    version = "42"
+    """
+
+    PYPROJECT_PACKAGE_DIR = """
+    [tool.setuptools]
+    package-dir = {"" = "src"}
+    """
+
+    @pytest.mark.parametrize(
+        "src_root, files",
+        [
+            (".", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),
+            (".", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),
+            ("src", {"setup.cfg": DALS(EXAMPLE_SETUPCFG)}),
+            ("src", {"pyproject.toml": DALS(EXAMPLE_PYPROJECT)}),
+            (
+                "src",
+                {
+                    "setup.cfg": DALS(EXAMPLE_SETUPCFG) + DALS(
+                        """
+                        packages = find:
+                        package_dir =
+                            =src
+
+                        [options.packages.find]
+                        where = src
+                        """
+                    )
+                }
+            ),
+            (
+                "src",
+                {
+                    "pyproject.toml": DALS(EXAMPLE_PYPROJECT) + DALS(
+                        """
+                        [tool.setuptools]
+                        package-dir = {"" = "src"}
+                        """
+                    )
+                },
+            ),
+        ]
+    )
+    def test_include_package_data(self, tmp_path, src_root, files):
+        """
+        Make sure auto-discovery does not affect package include_package_data.
+        See issue #3196.
+        """
+        jaraco.path.build(files, prefix=str(tmp_path))
+        self._simulate_package_with_data_files(tmp_path, src_root)
+
+        expected = {
+            os.path.normpath(f"{src_root}/proj/file1.txt").replace(os.sep, "/"),
+            os.path.normpath(f"{src_root}/proj/nested/file2.txt").replace(os.sep, "/"),
+        }
+
+        _run_build(tmp_path)
+
+        sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz")))
+        print("~~~~~ sdist_members ~~~~~")
+        print('\n'.join(sdist_files))
+        assert sdist_files >= expected
+
+        wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl")))
+        print("~~~~~ wheel_members ~~~~~")
+        print('\n'.join(wheel_files))
+        orig_files = {f.replace("src/", "").replace("lib/", "") for f in expected}
+        assert wheel_files >= orig_files
+
+
 def _populate_project_dir(root, files, options):
     # NOTE: Currently pypa/build will refuse to build the project if no
     # `pyproject.toml` or `setup.py` is found. So it is impossible to do
@@ -437,7 +533,7 @@ def _write_setupcfg(root, options):
 
 def _run_build(path, *flags):
     cmd = [sys.executable, "-m", "build", "--no-isolation", *flags, str(path)]
-    return run(cmd, env={'DISTUTILS_DEBUG': '1'})
+    return run(cmd, env={'DISTUTILS_DEBUG': ''})
 
 
 def _get_dist(dist_path, attrs):