Repository: psf/black

Issue:
Apply excludes before normalising symlinks
**Is your feature request related to a problem? Please describe.**

Our project contains symlinks like this:

```
project
+- a
   +- b -> ../x/y
   +- other files
+- x
   +- y
      +- lots of files
```

We would like to avoid black spending time formatting/complaining about the same files twice, so we make the excludes regex match `/a/b/`.
This doesn't help because the excludes are evaluated after normalising the path to `/x/y/`.
We cannot put that in the excludes because we do want to format the files.

**Describe the solution you'd like**

Excludes could be evaluated before normalising the path, or at least before evaluating symlinks.

**Describe alternatives you've considered**

Perhaps it would be best to evaluate excludes both before and after normalising the path.
There is also a use case for the way it is now, if we did want to exclude the files completely without listing all paths to them in the excludes.

Discussion:
github-actions[bot]: **diff-shades** reports zero changes comparing this PR (2a8804cb52e0134495ec7276d10a8ac2cd1c8133) to main (c6a031e623c7991ac9129f578dc21dffe2d7ede3).

---

[**What is this?**](https://black.readthedocs.io/en/latest/contributing/gauging_changes.html#diff-shades) | [Workflow run](https://github.com/psf/black/actions/runs/5909303241) | [diff-shades documentation](https://github.com/ichard26/diff-shades#readme)

Patch:
diff --git a/.coveragerc b/.coveragerc
deleted file mode 100644
index 5577e49..0000000
--- a/.coveragerc
+++ /dev/null
@@ -1,9 +0,0 @@
-[report]
-omit =
-  src/blib2to3/*
-  tests/data/*
-  */site-packages/*
-  .tox/*
-
-[run]
-relative_files = True
diff --git a/CHANGES.md b/CHANGES.md
index a14a55a..2168c1b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -20,6 +20,8 @@
 
 <!-- Changes to how Black can be configured -->
 
+- Black now applies exclusion and ignore logic before resolving symlinks (#3846)
+
 ### Packaging
 
 <!-- Changes to how Black is packaged, such as dependency requirements -->
diff --git a/README.md b/README.md
index 9d0b29a..b257c33 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
 <a href="https://coveralls.io/github/psf/black?branch=main"><img alt="Coverage Status" src="https://coveralls.io/repos/github/psf/black/badge.svg?branch=main"></a>
 <a href="https://github.com/psf/black/blob/main/LICENSE"><img alt="License: MIT" src="https://black.readthedocs.io/en/stable/_static/license.svg"></a>
 <a href="https://pypi.org/project/black/"><img alt="PyPI" src="https://img.shields.io/pypi/v/black"></a>
-<a href="https://pepy.tech/project/black"><img alt="Downloads" src="https://pepy.tech/badge/black"></a>
+<a href="https://pepy.tech/project/black"><img alt="Downloads" src="https://static.pepy.tech/badge/black"></a>
 <a href="https://anaconda.org/conda-forge/black/"><img alt="conda-forge" src="https://img.shields.io/conda/dn/conda-forge/black.svg?label=conda-forge"></a>
 <a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
 </p>
diff --git a/docs/conf.py b/docs/conf.py
index 7fc4f8f..f7cf1b4 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -213,4 +213,4 @@ def make_pypi_svg(version: str) -> None:
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {"https://docs.python.org/3/": None}
+intersphinx_mapping = {"<name>": ("https://docs.python.org/3/", None)}
diff --git a/docs/requirements.txt b/docs/requirements.txt
index ff179f3..dad39f6 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,9 +1,9 @@
 # Used by ReadTheDocs; pinned requirements for stability.
 
 myst-parser==2.0.0
-Sphinx==6.1.3
+Sphinx==7.2.3
 # Older versions break Sphinx even though they're declared to be supported.
 docutils==0.19
 sphinxcontrib-programoutput==0.17
 sphinx_copybutton==0.5.2
-furo==2023.7.26
+furo==2023.8.19
diff --git a/pyproject.toml b/pyproject.toml
index 6cd3f34..ea5c9f8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -210,3 +210,12 @@ filterwarnings = [
     # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12
     '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning'''
 ]
+[tool.coverage.report]
+omit = [
+  "src/blib2to3/*",
+  "tests/data/*",
+  "*/site-packages/*",
+  ".tox/*"
+]
+[tool.coverage.run]
+relative_files = true
diff --git a/src/black/__init__.py b/src/black/__init__.py
index dc06eab..6fc91d2 100644
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -560,9 +560,10 @@ def main(  # noqa: C901
             content=code, fast=fast, write_back=write_back, mode=mode, report=report
         )
     else:
+        assert root is not None  # root is only None if code is not None
         try:
             sources = get_sources(
-                ctx=ctx,
+                root=root,
                 src=src,
                 quiet=quiet,
                 verbose=verbose,
@@ -615,7 +616,7 @@ def main(  # noqa: C901
 
 def get_sources(
     *,
-    ctx: click.Context,
+    root: Path,
     src: Tuple[str, ...],
     quiet: bool,
     verbose: bool,
@@ -628,7 +629,6 @@ def get_sources(
 ) -> Set[Path]:
     """Compute the set of files to be formatted."""
     sources: Set[Path] = set()
-    root = ctx.obj["root"]
 
     using_default_exclude = exclude is None
     exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude
@@ -645,7 +645,7 @@ def get_sources(
 
         if is_stdin or p.is_file():
             normalized_path: Optional[str] = normalize_path_maybe_ignore(
-                p, ctx.obj["root"], report
+                p, root, report
             )
             if normalized_path is None:
                 if verbose:
@@ -674,7 +674,9 @@ def get_sources(
 
             sources.add(p)
         elif p.is_dir():
-            p = root / normalize_path_maybe_ignore(p, ctx.obj["root"], report)
+            p_relative = normalize_path_maybe_ignore(p, root, report)
+            assert p_relative is not None
+            p = root / p_relative
             if verbose:
                 out(f'Found input source directory: "{p}"', fg="blue")
 
@@ -686,7 +688,7 @@ def get_sources(
             sources.update(
                 gen_python_files(
                     p.iterdir(),
-                    ctx.obj["root"],
+                    root,
                     include,
                     exclude,
                     extend_exclude,
diff --git a/src/black/files.py b/src/black/files.py
index 368e417..362898d 100644
--- a/src/black/files.py
+++ b/src/black/files.py
@@ -330,35 +330,37 @@ def gen_python_files(
 
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     for child in paths:
-        normalized_path = normalize_path_maybe_ignore(child, root, report)
-        if normalized_path is None:
-            continue
+        root_relative_path = child.absolute().relative_to(root).as_posix()
 
         # First ignore files matching .gitignore, if passed
         if gitignore_dict and _path_is_ignored(
-            normalized_path, root, gitignore_dict, report
+            root_relative_path, root, gitignore_dict, report
         ):
             continue
 
         # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
-        normalized_path = "/" + normalized_path
+        root_relative_path = "/" + root_relative_path
         if child.is_dir():
-            normalized_path += "/"
+            root_relative_path += "/"
 
-        if path_is_excluded(normalized_path, exclude):
+        if path_is_excluded(root_relative_path, exclude):
             report.path_ignored(child, "matches the --exclude regular expression")
             continue
 
-        if path_is_excluded(normalized_path, extend_exclude):
+        if path_is_excluded(root_relative_path, extend_exclude):
             report.path_ignored(
                 child, "matches the --extend-exclude regular expression"
             )
             continue
 
-        if path_is_excluded(normalized_path, force_exclude):
+        if path_is_excluded(root_relative_path, force_exclude):
             report.path_ignored(child, "matches the --force-exclude regular expression")
             continue
 
+        normalized_path = normalize_path_maybe_ignore(child, root, report)
+        if normalized_path is None:
+            continue
+
         if child.is_dir():
             # If gitignore is None, gitignore usage is disabled, while a Falsey
             # gitignore is when the directory doesn't have a .gitignore file.


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8a13938..7daa31e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -45,11 +45,12 @@ jobs:
       - name: Install tox
         run: |
           python -m pip install --upgrade pip
-          python -m pip install --upgrade 'tox<4.7'
+          python -m pip install --upgrade tox
 
       - name: Unit tests
         if: "!startsWith(matrix.python-version, 'pypy')"
-        run: tox -e ci-py -- -v --color=yes
+        run:
+          tox -e ci-py$(echo ${{ matrix.python-version }} | tr -d '.') -- -v --color=yes
 
       - name: Unit tests (pypy)
         if: "startsWith(matrix.python-version, 'pypy')"
@@ -61,7 +62,7 @@ jobs:
         if:
           github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' &&
           !startsWith(matrix.python-version, 'pypy')
-        uses: AndreMiras/coveralls-python-action@v20201129
+        uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           parallel: true
@@ -76,7 +77,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Send finished signal to Coveralls
-        uses: AndreMiras/coveralls-python-action@v20201129
+        uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99
         with:
           parallel-finished: true
           debug: true
diff --git a/tests/test_black.py b/tests/test_black.py
index 8ae9217..4fb6aef 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -492,9 +492,7 @@ def test_false_positive_symlink_output_issue_3384(self) -> None:
         project_root = Path(THIS_DIR / "data" / "nested_gitignore_tests")
         working_directory = project_root / "root"
         target_abspath = working_directory / "child"
-        target_contents = (
-            src.relative_to(working_directory) for src in target_abspath.iterdir()
-        )
+        target_contents = list(target_abspath.iterdir())
 
         def mock_n_calls(responses: List[bool]) -> Callable[[], bool]:
             def _mocked_calls() -> bool:
@@ -507,13 +505,11 @@ def _mocked_calls() -> bool:
         with patch("pathlib.Path.iterdir", return_value=target_contents), patch(
             "pathlib.Path.cwd", return_value=working_directory
         ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])):
-            ctx = FakeContext()
             # Note that the root folder (project_root) isn't the folder
             # named "root" (aka working_directory)
-            ctx.obj["root"] = project_root
             report = MagicMock(verbose=True)
             black.get_sources(
-                ctx=ctx,
+                root=project_root,
                 src=("./child",),
                 quiet=False,
                 verbose=True,
@@ -2163,7 +2159,7 @@ def assert_collected_sources(
     src: Sequence[Union[str, Path]],
     expected: Sequence[Union[str, Path]],
     *,
-    ctx: Optional[FakeContext] = None,
+    root: Optional[Path] = None,
     exclude: Optional[str] = None,
     include: Optional[str] = None,
     extend_exclude: Optional[str] = None,
@@ -2179,7 +2175,7 @@ def assert_collected_sources(
     )
     gs_force_exclude = None if force_exclude is None else compile_pattern(force_exclude)
     collected = black.get_sources(
-        ctx=ctx or FakeContext(),
+        root=root or THIS_DIR,
         src=gs_src,
         quiet=False,
         verbose=False,
@@ -2215,9 +2211,7 @@ def test_gitignore_used_as_default(self) -> None:
             base / "b/.definitely_exclude/a.pyi",
         ]
         src = [base / "b/"]
-        ctx = FakeContext()
-        ctx.obj["root"] = base
-        assert_collected_sources(src, expected, ctx=ctx, extend_exclude=r"/exclude/")
+        assert_collected_sources(src, expected, root=base, extend_exclude=r"/exclude/")
 
     def test_gitignore_used_on_multiple_sources(self) -> None:
         root = Path(DATA_DIR / "gitignore_used_on_multiple_sources")
@@ -2225,10 +2219,8 @@ def test_gitignore_used_on_multiple_sources(self) -> None:
             root / "dir1" / "b.py",
             root / "dir2" / "b.py",
         ]
-        ctx = FakeContext()
-        ctx.obj["root"] = root
         src = [root / "dir1", root / "dir2"]
-        assert_collected_sources(src, expected, ctx=ctx)
+        assert_collected_sources(src, expected, root=root)
 
     @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
     def test_exclude_for_issue_1572(self) -> None:
@@ -2334,9 +2326,7 @@ def test_gitignore_that_ignores_subfolders(self) -> None:
         # If gitignore with */* is in root
         root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests" / "subdir")
         expected = [root / "b.py"]
-        ctx = FakeContext()
-        ctx.obj["root"] = root
-        assert_collected_sources([root], expected, ctx=ctx)
+        assert_collected_sources([root], expected, root=root)
 
         # If .gitignore with */* is nested
         root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests")
@@ -2344,17 +2334,13 @@ def test_gitignore_that_ignores_subfolders(self) -> None:
             root / "a.py",
             root / "subdir" / "b.py",
         ]
-        ctx = FakeContext()
-        ctx.obj["root"] = root
-        assert_collected_sources([root], expected, ctx=ctx)
+        assert_collected_sources([root], expected, root=root)
 
         # If command is executed from outer dir
         root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests")
         target = root / "subdir"
         expected = [target / "b.py"]
-        ctx = FakeContext()
-        ctx.obj["root"] = root
-        assert_collected_sources([target], expected, ctx=ctx)
+        assert_collected_sources([target], expected, root=root)
 
     def test_empty_include(self) -> None:
         path = DATA_DIR / "include_exclude_tests"
@@ -2387,38 +2373,48 @@ def test_extend_exclude(self) -> None:
         )
 
     @pytest.mark.incompatible_with_mypyc
-    def test_symlink_out_of_root_directory(self) -> None:
+    def test_symlinks(self) -> None:
         path = MagicMock()
         root = THIS_DIR.resolve()
-        child = MagicMock()
         include = re.compile(black.DEFAULT_INCLUDES)
         exclude = re.compile(black.DEFAULT_EXCLUDES)
         report = black.Report()
         gitignore = PathSpec.from_lines("gitwildmatch", [])
-        # `child` should behave like a symlink which resolved path is clearly
-        # outside of the `root` directory.
-        path.iterdir.return_value = [child]
-        child.resolve.return_value = Path("/a/b/c")
-        child.as_posix.return_value = "/a/b/c"
-        try:
-            list(
-                black.gen_python_files(
-                    path.iterdir(),
-                    root,
-                    include,
-                    exclude,
-                    None,
-                    None,
-                    report,
-                    {path: gitignore},
-                    verbose=False,
-                    quiet=False,
-                )
+
+        regular = MagicMock()
+        outside_root_symlink = MagicMock()
+        ignored_symlink = MagicMock()
+
+        path.iterdir.return_value = [regular, outside_root_symlink, ignored_symlink]
+
+        regular.absolute.return_value = root / "regular.py"
+        regular.resolve.return_value = root / "regular.py"
+        regular.is_dir.return_value = False
+
+        outside_root_symlink.absolute.return_value = root / "symlink.py"
+        outside_root_symlink.resolve.return_value = Path("/nowhere")
+
+        ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py"
+
+        files = list(
+            black.gen_python_files(
+                path.iterdir(),
+                root,
+                include,
+                exclude,
+                None,
+                None,
+                report,
+                {path: gitignore},
+                verbose=False,
+                quiet=False,
             )
-        except ValueError as ve:
-            pytest.fail(f"`get_python_files_in_dir()` failed: {ve}")
+        )
+        assert files == [regular]
+
         path.iterdir.assert_called_once()
-        child.resolve.assert_called_once()
+        outside_root_symlink.resolve.assert_called_once()
+        ignored_symlink.resolve.assert_not_called()
 
     @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None))
     def test_get_sources_with_stdin(self) -> None: