Repository: pypa/setuptools

Issue:
header files not included in tar.gz file
`python3 -m build` is using `sdist` to create the file containing the sources. 
Unfortunately, `.h` files are not included by default by sdist, though `.c` files are.
The `depends` member of `Extension` is ignored by sdist. 

This is really annoying since we are then required to create a MANIFEST.in file. 

Could `setuptools` do something to avoid this limitation of `sdist` ?  


Discussion:
Hi @chmike, thank you very much for bringing this topic up!

Your problem description makes a lot of sense to me.
Would you be interested in providing a PR to add support for such a feature?

Just as a reference, these pieces of code might be relevant:

- [_distutils.command.sdist](https://github.com/pypa/setuptools/blob/1a1397a4ef0327ba09ddd4a3d0916af48d3be7ec/setuptools/_distutils/command/sdist.py#L210)
- [setuptools.command.sdist](https://github.com/pypa/setuptools/blob/1a1397a4ef0327ba09ddd4a3d0916af48d3be7ec/setuptools/command/sdist.py#L103)
- [setuptools.command.egg_info/manifest_maker](https://github.com/pypa/setuptools/blob/1a1397a4ef0327ba09ddd4a3d0916af48d3be7ec/setuptools/command/egg_info.py#L577)

Resolving patch:
diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py
--- a/setuptools/command/build_ext.py
+++ b/setuptools/command/build_ext.py
@@ -4,6 +4,7 @@
 from importlib.machinery import EXTENSION_SUFFIXES
 from importlib.util import cache_from_source as _compiled_file_name
 from typing import Dict, Iterator, List, Tuple
+from pathlib import Path
 
 from distutils.command.build_ext import build_ext as _du_build_ext
 from distutils.ccompiler import new_compiler
@@ -261,6 +262,47 @@ def links_to_dynamic(self, ext):
         pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])
         return any(pkg + libname in libnames for libname in ext.libraries)
 
+    def get_source_files(self) -> List[str]:
+        return [*_build_ext.get_source_files(self), *self._get_internal_depends()]
+
+    def _get_internal_depends(self) -> Iterator[str]:
+        """Yield ``ext.depends`` that are contained by the project directory"""
+        project_root = Path(self.distribution.src_root or os.curdir).resolve()
+        depends = (dep for ext in self.extensions for dep in ext.depends)
+
+        def skip(orig_path: str, reason: str) -> None:
+            log.info(
+                "dependency %s won't be automatically "
+                "included in the manifest: the path %s",
+                orig_path,
+                reason,
+            )
+
+        for dep in depends:
+            path = Path(dep)
+
+            if path.is_absolute():
+                skip(dep, "must be relative")
+                continue
+
+            if ".." in path.parts:
+                skip(dep, "can't have `..` segments")
+                continue
+
+            try:
+                resolved = (project_root / path).resolve(strict=True)
+            except OSError:
+                skip(dep, "doesn't exist")
+                continue
+
+            try:
+                resolved.relative_to(project_root)
+            except ValueError:
+                skip(dep, "must be inside the project root")
+                continue
+
+            yield path.as_posix()
+
     def get_outputs(self) -> List[str]:
         if self.inplace:
             return list(self.get_output_mapping().keys())


Test patch:
diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py
--- a/setuptools/tests/test_sdist.py
+++ b/setuptools/tests/test_sdist.py
@@ -6,19 +6,28 @@
 import unicodedata
 import contextlib
 import io
+import tarfile
+import logging
+import distutils
+from inspect import cleandoc
+from pathlib import Path
 from unittest import mock
 
 import pytest
 
+from distutils.core import run_setup
 from setuptools import Command
 from setuptools._importlib import metadata
 from setuptools import SetuptoolsDeprecationWarning
 from setuptools.command.sdist import sdist
 from setuptools.command.egg_info import manifest_maker
 from setuptools.dist import Distribution
+from setuptools.extension import Extension
 from setuptools.tests import fail_on_ascii
 from .text import Filenames
 
+import jaraco.path
+
 
 SETUP_ATTRS = {
     'name': 'sdist_test',
@@ -37,6 +46,13 @@
     % SETUP_ATTRS
 )
 
+EXTENSION = Extension(
+    name="sdist_test.f",
+    sources=[os.path.join("sdist_test", "f.c")],
+    depends=[os.path.join("sdist_test", "f.h")],
+)
+EXTENSION_SOURCES = EXTENSION.sources + EXTENSION.depends
+
 
 @contextlib.contextmanager
 def quiet():
@@ -93,15 +109,33 @@ def latin1_fail():
     "os.environ.get('PYTEST_XDIST_WORKER')",
     reason="pytest-dev/pytest-xdist#843",
 )
+skip_under_stdlib_distutils = pytest.mark.skipif(
+    not distutils.__package__.startswith('setuptools'),
+    reason="the test is not supported with stdlib distutils",
+)
 
 
 def touch(path):
+    if isinstance(path, str):
+        path = Path(path)
     path.write_text('', encoding='utf-8')
+    return path
+
+
+def symlink_or_skip_test(src, dst):
+    try:
+        os.symlink(src, dst)
+        return dst
+    except (OSError, NotImplementedError):
+        pytest.skip("symlink not supported in OS")
 
 
 class TestSdistTest:
     @pytest.fixture(autouse=True)
     def source_dir(self, tmpdir):
+        tmpdir = tmpdir / "project_root"
+        tmpdir.mkdir()
+
         (tmpdir / 'setup.py').write_text(SETUP_PY, encoding='utf-8')
 
         # Set up the rest of the test package
@@ -114,9 +148,13 @@ def source_dir(self, tmpdir):
         for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
             touch(test_pkg / fname)
         touch(data_folder / 'e.dat')
+        # C sources are not included by default, but they will be,
+        # if an extension module uses them as sources or depends
+        for fname in EXTENSION_SOURCES:
+            touch(tmpdir / fname)
 
         with tmpdir.as_cwd():
-            yield
+            yield tmpdir
 
     def assert_package_data_in_manifest(self, cmd):
         manifest = cmd.filelist.files
@@ -125,6 +163,19 @@ def assert_package_data_in_manifest(self, cmd):
         assert os.path.join('sdist_test', 'c.rst') not in manifest
         assert os.path.join('d', 'e.dat') in manifest
 
+    def setup_with_extension(self):
+        setup_attrs = {**SETUP_ATTRS, 'ext_modules': [EXTENSION]}
+
+        dist = Distribution(setup_attrs)
+        dist.script_name = 'setup.py'
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+
+        with quiet():
+            cmd.run()
+
+        return cmd
+
     def test_package_data_in_sdist(self):
         """Regression test for pull request #4: ensures that files listed in
         package_data are included in the manifest even if they're not added to
@@ -159,6 +210,117 @@ def test_package_data_and_include_package_data_in_sdist(self):
 
         self.assert_package_data_in_manifest(cmd)
 
+    def test_extension_sources_in_sdist(self):
+        """
+        Ensure that the files listed in Extension.sources and Extension.depends
+        are automatically included in the manifest.
+        """
+        cmd = self.setup_with_extension()
+        self.assert_package_data_in_manifest(cmd)
+        manifest = cmd.filelist.files
+        for path in EXTENSION_SOURCES:
+            assert path in manifest
+
+    def test_missing_extension_sources(self):
+        """
+        Similar to test_extension_sources_in_sdist but the referenced files don't exist.
+        Missing files should not be included in distribution (with no error raised).
+        """
+        for path in EXTENSION_SOURCES:
+            os.remove(path)
+
+        cmd = self.setup_with_extension()
+        self.assert_package_data_in_manifest(cmd)
+        manifest = cmd.filelist.files
+        for path in EXTENSION_SOURCES:
+            assert path not in manifest
+
+    def test_symlinked_extension_sources(self):
+        """
+        Similar to test_extension_sources_in_sdist but the referenced files are
+        instead symbolic links to project-local files. Referenced file paths
+        should be included. Symlink targets themselves should NOT be included.
+        """
+        symlinked = []
+        for path in EXTENSION_SOURCES:
+            base, ext = os.path.splitext(path)
+            target = base + "_target." + ext
+
+            os.rename(path, target)
+            symlink_or_skip_test(os.path.basename(target), path)
+            symlinked.append(target)
+
+        cmd = self.setup_with_extension()
+        self.assert_package_data_in_manifest(cmd)
+        manifest = cmd.filelist.files
+        for path in EXTENSION_SOURCES:
+            assert path in manifest
+        for path in symlinked:
+            assert path not in manifest
+
+    _INVALID_PATHS = {
+        "must be relative": lambda: (
+            os.path.abspath(os.path.join("sdist_test", "f.h"))
+        ),
+        "can't have `..` segments": lambda: (
+            os.path.join("sdist_test", "..", "sdist_test", "f.h")
+        ),
+        "doesn't exist": lambda: (
+            os.path.join("sdist_test", "this_file_does_not_exist.h")
+        ),
+        "must be inside the project root": lambda: (
+            symlink_or_skip_test(
+                touch(os.path.join("..", "outside_of_project_root.h")),
+                "symlink.h",
+            )
+        ),
+    }
+
+    @skip_under_stdlib_distutils
+    @pytest.mark.parametrize("reason", _INVALID_PATHS.keys())
+    def test_invalid_extension_depends(self, reason, caplog):
+        """
+        Due to backwards compatibility reasons, `Extension.depends` should accept
+        invalid/weird paths, but then ignore them when building a sdist.
+
+        This test verifies that the source distribution is still built
+        successfully with such paths, but that instead of adding these paths to
+        the manifest, we emit an informational message, notifying the user that
+        the invalid path won't be automatically included.
+        """
+        invalid_path = self._INVALID_PATHS[reason]()
+        extension = Extension(
+            name="sdist_test.f",
+            sources=[],
+            depends=[invalid_path],
+        )
+        setup_attrs = {**SETUP_ATTRS, 'ext_modules': [extension]}
+
+        dist = Distribution(setup_attrs)
+        dist.script_name = 'setup.py'
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+
+        with quiet(), caplog.at_level(logging.INFO):
+            cmd.run()
+
+        self.assert_package_data_in_manifest(cmd)
+        manifest = cmd.filelist.files
+        assert invalid_path not in manifest
+
+        expected_message = [
+            message
+            for (logger, level, message) in caplog.record_tuples
+            if (
+                logger == "root"  #
+                and level == logging.INFO  #
+                and invalid_path in message  #
+            )
+        ]
+        assert len(expected_message) == 1
+        (expected_message,) = expected_message
+        assert reason in expected_message
+
     def test_custom_build_py(self):
         """
         Ensure projects defining custom build_py don't break
@@ -238,14 +400,14 @@ def test_setup_py_excluded(self):
         manifest = cmd.filelist.files
         assert 'setup.py' not in manifest
 
-    def test_defaults_case_sensitivity(self, tmpdir):
+    def test_defaults_case_sensitivity(self, source_dir):
         """
         Make sure default files (README.*, etc.) are added in a case-sensitive
         way to avoid problems with packages built on Windows.
         """
 
-        touch(tmpdir / 'readme.rst')
-        touch(tmpdir / 'SETUP.cfg')
+        touch(source_dir / 'readme.rst')
+        touch(source_dir / 'SETUP.cfg')
 
         dist = Distribution(SETUP_ATTRS)
         # the extension deliberately capitalized for this test
@@ -527,15 +689,15 @@ def test_sdist_with_latin1_encoded_filename(self):
     }
 
     @pytest.mark.parametrize("config", _EXAMPLE_DIRECTIVES.keys())
-    def test_add_files_referenced_by_config_directives(self, tmp_path, config):
+    def test_add_files_referenced_by_config_directives(self, source_dir, config):
         config_file, _, _ = config.partition(" - ")
         config_text = self._EXAMPLE_DIRECTIVES[config]
-        (tmp_path / 'src').mkdir()
-        (tmp_path / 'src/VERSION.txt').write_text("0.42", encoding="utf-8")
-        (tmp_path / 'README.rst').write_text("hello world!", encoding="utf-8")
-        (tmp_path / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
-        (tmp_path / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
-        (tmp_path / config_file).write_text(config_text, encoding="utf-8")
+        (source_dir / 'src').mkdir()
+        (source_dir / 'src/VERSION.txt').write_text("0.42", encoding="utf-8")
+        (source_dir / 'README.rst').write_text("hello world!", encoding="utf-8")
+        (source_dir / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
+        (source_dir / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
+        (source_dir / config_file).write_text(config_text, encoding="utf-8")
 
         dist = Distribution({"packages": []})
         dist.script_name = 'setup.py'
@@ -555,11 +717,11 @@ def test_add_files_referenced_by_config_directives(self, tmp_path, config):
         assert '/' not in cmd.filelist.files
         assert '\\' not in cmd.filelist.files
 
-    def test_pyproject_toml_in_sdist(self, tmpdir):
+    def test_pyproject_toml_in_sdist(self, source_dir):
         """
         Check if pyproject.toml is included in source distribution if present
         """
-        touch(tmpdir / 'pyproject.toml')
+        touch(source_dir / 'pyproject.toml')
         dist = Distribution(SETUP_ATTRS)
         dist.script_name = 'setup.py'
         cmd = sdist(dist)
@@ -569,11 +731,11 @@ def test_pyproject_toml_in_sdist(self, tmpdir):
         manifest = cmd.filelist.files
         assert 'pyproject.toml' in manifest
 
-    def test_pyproject_toml_excluded(self, tmpdir):
+    def test_pyproject_toml_excluded(self, source_dir):
         """
         Check that pyproject.toml can excluded even if present
         """
-        touch(tmpdir / 'pyproject.toml')
+        touch(source_dir / 'pyproject.toml')
         with open('MANIFEST.in', 'w') as mts:
             print('exclude pyproject.toml', file=mts)
         dist = Distribution(SETUP_ATTRS)
@@ -585,8 +747,8 @@ def test_pyproject_toml_excluded(self, tmpdir):
         manifest = cmd.filelist.files
         assert 'pyproject.toml' not in manifest
 
-    def test_build_subcommand_source_files(self, tmpdir):
-        touch(tmpdir / '.myfile~')
+    def test_build_subcommand_source_files(self, source_dir):
+        touch(source_dir / '.myfile~')
 
         # Sanity check: without custom commands file list should not be affected
         dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"})
@@ -645,3 +807,109 @@ def test_default_revctrl():
     )
     res = ep.load()
     assert hasattr(res, '__iter__')
+
+
+class TestRegressions:
+    """
+    Can be removed/changed if the project decides to change how it handles symlinks
+    or external files.
+    """
+
+    @staticmethod
+    def files_for_symlink_in_extension_depends(tmp_path, dep_path):
+        return {
+            "external": {
+                "dir": {"file.h": ""},
+            },
+            "project": {
+                "setup.py": cleandoc(
+                    f"""
+                    from setuptools import Extension, setup
+                    setup(
+                        name="myproj",
+                        version="42",
+                        ext_modules=[
+                            Extension(
+                                "hello", sources=["hello.pyx"],
+                                depends=[{dep_path!r}]
+                            )
+                        ],
+                    )
+                    """
+                ),
+                "hello.pyx": "",
+                "MANIFEST.in": "global-include *.h",
+            },
+        }
+
+    @pytest.mark.parametrize(
+        "dep_path", ("myheaders/dir/file.h", "myheaders/dir/../dir/file.h")
+    )
+    def test_symlink_in_extension_depends(self, monkeypatch, tmp_path, dep_path):
+        # Given a project with a symlinked dir and a "depends" targeting that dir
+        files = self.files_for_symlink_in_extension_depends(tmp_path, dep_path)
+        jaraco.path.build(files, prefix=str(tmp_path))
+        symlink_or_skip_test(tmp_path / "external", tmp_path / "project/myheaders")
+
+        # When `sdist` runs, there should be no error
+        members = run_sdist(monkeypatch, tmp_path / "project")
+        # and the sdist should contain the symlinked files
+        for expected in (
+            "myproj-42/hello.pyx",
+            "myproj-42/myheaders/dir/file.h",
+        ):
+            assert expected in members
+
+    @staticmethod
+    def files_for_external_path_in_extension_depends(tmp_path, dep_path):
+        head, _, tail = dep_path.partition("$tmp_path$/")
+        dep_path = tmp_path / tail if tail else head
+
+        return {
+            "external": {
+                "dir": {"file.h": ""},
+            },
+            "project": {
+                "setup.py": cleandoc(
+                    f"""
+                    from setuptools import Extension, setup
+                    setup(
+                        name="myproj",
+                        version="42",
+                        ext_modules=[
+                            Extension(
+                                "hello", sources=["hello.pyx"],
+                                depends=[{str(dep_path)!r}]
+                            )
+                        ],
+                    )
+                    """
+                ),
+                "hello.pyx": "",
+                "MANIFEST.in": "global-include *.h",
+            },
+        }
+
+    @pytest.mark.parametrize(
+        "dep_path", ("$tmp_path$/external/dir/file.h", "../external/dir/file.h")
+    )
+    def test_external_path_in_extension_depends(self, monkeypatch, tmp_path, dep_path):
+        # Given a project with a "depends" targeting an external dir
+        files = self.files_for_external_path_in_extension_depends(tmp_path, dep_path)
+        jaraco.path.build(files, prefix=str(tmp_path))
+        # When `sdist` runs, there should be no error
+        members = run_sdist(monkeypatch, tmp_path / "project")
+        # and the sdist should not contain the external file
+        for name in members:
+            assert "file.h" not in name
+
+
+def run_sdist(monkeypatch, project):
+    """Given a project directory, run the sdist and return its contents"""
+    monkeypatch.chdir(project)
+    with quiet():
+        run_setup("setup.py", ["sdist"])
+
+    archive = next((project / "dist").glob("*.tar.gz"))
+    with tarfile.open(str(archive)) as tar:
+        return set(tar.getnames())