Repository: ambv/black
Pull request #281 (2018-05-30 21:08:56)

Added --include and --exclude cli options
These 2 options allow you to pass in regular expressions that determine whether files/directories are included or excluded in the recursive search.

This should address #270. I put the change log entry for this under `18.6b0` since I don't think it has been released yet - but let me know if this was incorrect! :sparkles: 

Autophagy: Also, I genuinely do not know how I would check the types of `exclude` and `include` here https://github.com/ambv/black/pull/281/files#diff-a75051ebf8e3452bf3d590c186944252R2791 since I am passing in compiled regular expressions which have a type of `_sre.SRE_Pattern`, but apparently `AttributeError: module '_sre' has no attribute 'SRE_Pattern'` :sweat_smile: 
JelleZijlstra: Try `typing.Pattern[str]`.
coveralls: ## Pull Request Test Coverage Report for [Build 430](https://coveralls.io/builds/17240338)

* **11** of **18**   **(61.11%)**  changed or added relevant lines in **1** file are covered.
* **1** unchanged line in **1** file lost coverage.
* Overall coverage decreased (**-0.2%**) to **95.805%**

---

|  Changes Missing Coverage | Covered Lines | Changed/Added Lines | % |
| :-----|--------------|--------|---: |
| [black.py](https://coveralls.io/builds/17240338/source?filename=black.py#L246) | 11 | 18 | 61.11%
<!-- | **Total:** | **11** | **18** | **61.11%** | -->

|  Files with Coverage Reduction | New Missed Lines | % |
| :-----|--------------|--: |
| [black.py](https://coveralls.io/builds/17240338/source?filename=black.py#L2799) | 1 | 94.76% |
<!-- | **Total:** | **1** |  | -->

|  Totals | [![Coverage Status](https://coveralls.io/builds/17240338/badge)](https://coveralls.io/builds/17240338) |
| :-- | --: |
| Change from base [Build 429](https://coveralls.io/builds/17238932): |  -0.2% |
| Covered Lines: | 2238 |
| Relevant Lines: | 2336 |

---
##### 💛  - [Coveralls](https://coveralls.io)

Autophagy: @JelleZijlstra Ah! Thank you a bunch.
autophagy: @ambv Added some logic to allow for prepended directories in the filter, and confirmed it works well with both `*` and `.`. I'm building up the path to search from using the children, rather than the path, as it's a little easier when the path itself is something like `.`.
ambv: Makes sense to just mention: "use forward-slashes on Windows, too."

Patch:
diff --git a/README.md b/README.md
index 12af03f..79b8347 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,17 @@ Options:
                               **kwargs.  [default: per-file auto-detection]
   -S, --skip-string-normalization
                               Don't normalize string quotes or prefixes.
+  --include TEXT              A regular expression that matches files and
+                              directories that should be included on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default: \.pyi?$]
+  --exclude TEXT              A regular expression that matches files and
+                              directories that should be excluded on
+                              recursive searches. On Windows, use forward
+                              slashes for directories.  [default:
+                              build/|buck-out/|dist/|_build/|\.git/|\.hg/|
+                              \.mypy_cache/|\.tox/|\.venv/]
+
   --version                   Show the version and exit.
   --help                      Show this message and exit.
 ```
@@ -698,6 +709,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.6b0
 
+* added `--include` and `--exclude` (#270)
+
 * added `--skip-string-normalization` (#118)
 
 * fixed stdin handling not working correctly if an old version of Click was
diff --git a/black.py b/black.py
index 4599bdd..ce4a44f 100644
--- a/black.py
+++ b/black.py
@@ -46,6 +46,10 @@
 
 __version__ = "18.5b1"
 DEFAULT_LINE_LENGTH = 88
+DEFAULT_EXCLUDES = (
+    r"build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/"
+)
+DEFAULT_INCLUDES = r"\.pyi?$"
 CACHE_DIR = Path(user_cache_dir("black", version=__version__))
 
 
@@ -189,6 +193,28 @@ class FileMode(Flag):
     is_flag=True,
     help="Don't normalize string quotes or prefixes.",
 )
+@click.option(
+    "--include",
+    type=str,
+    default=DEFAULT_INCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "included on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)
+@click.option(
+    "--exclude",
+    type=str,
+    default=DEFAULT_EXCLUDES,
+    help=(
+        "A regular expression that matches files and directories that should be "
+        "excluded on recursive searches. On Windows, use forward slashes for "
+        "directories."
+    ),
+    show_default=True,
+)
 @click.version_option(version=__version__)
 @click.argument(
     "src",
@@ -208,14 +234,26 @@ def main(
     py36: bool,
     skip_string_normalization: bool,
     quiet: bool,
+    include: str,
+    exclude: str,
     src: List[str],
 ) -> None:
     """The uncompromising code formatter."""
     sources: List[Path] = []
+    try:
+        include_regex = re.compile(include)
+    except re.error:
+        err(f"Invalid regular expression for include given: {include!r}")
+        ctx.exit(2)
+    try:
+        exclude_regex = re.compile(exclude)
+    except re.error:
+        err(f"Invalid regular expression for exclude given: {exclude!r}")
+        ctx.exit(2)
     for s in src:
         p = Path(s)
         if p.is_dir():
-            sources.extend(gen_python_files_in_dir(p))
+            sources.extend(gen_python_files_in_dir(p, include_regex, exclude_regex))
         elif p.is_file():
             # if a file was explicitly given, we don't care about its extension
             sources.append(p)
@@ -2750,33 +2788,35 @@ def get_future_imports(node: Node) -> Set[str]:
     return imports
 
 
-PYTHON_EXTENSIONS = {".py", ".pyi"}
-BLACKLISTED_DIRECTORIES = {
-    "build",
-    "buck-out",
-    "dist",
-    "_build",
-    ".git",
-    ".hg",
-    ".mypy_cache",
-    ".tox",
-    ".venv",
-}
-
-
-def gen_python_files_in_dir(path: Path) -> Iterator[Path]:
-    """Generate all files under `path` which aren't under BLACKLISTED_DIRECTORIES
-    and have one of the PYTHON_EXTENSIONS.
+def gen_python_files_in_dir(
+    path: Path, include: Pattern[str], exclude: Pattern[str]
+) -> Iterator[Path]:
+    """Generate all files under `path` whose paths are not excluded by the
+    `exclude` regex, but are included by the `include` regex.
     """
+
     for child in path.iterdir():
+        searchable_path = str(child.as_posix())
+        if Path(child.parts[0]).is_dir():
+            searchable_path = "/" + searchable_path
         if child.is_dir():
-            if child.name in BLACKLISTED_DIRECTORIES:
+            searchable_path = searchable_path + "/"
+            exclude_match = exclude.search(searchable_path)
+            if exclude_match and len(exclude_match.group()) > 0:
                 continue
 
-            yield from gen_python_files_in_dir(child)
+            yield from gen_python_files_in_dir(child, include, exclude)
 
-        elif child.is_file() and child.suffix in PYTHON_EXTENSIONS:
-            yield child
+        else:
+            include_match = include.search(searchable_path)
+            exclude_match = exclude.search(searchable_path)
+            if (
+                child.is_file()
+                and include_match
+                and len(include_match.group()) > 0
+                and (not exclude_match or len(exclude_match.group()) == 0)
+            ):
+                yield child
 
 
 @dataclass


Test patch:
diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.pie b/tests/include_exclude_tests/b/.definitely_exclude/a.pie
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.py b/tests/include_exclude_tests/b/.definitely_exclude/a.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/.definitely_exclude/a.pyi b/tests/include_exclude_tests/b/.definitely_exclude/a.pyi
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/dont_exclude/a.pie b/tests/include_exclude_tests/b/dont_exclude/a.pie
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/dont_exclude/a.py b/tests/include_exclude_tests/b/dont_exclude/a.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/dont_exclude/a.pyi b/tests/include_exclude_tests/b/dont_exclude/a.pyi
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/exclude/a.pie b/tests/include_exclude_tests/b/exclude/a.pie
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/exclude/a.py b/tests/include_exclude_tests/b/exclude/a.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/include_exclude_tests/b/exclude/a.pyi b/tests/include_exclude_tests/b/exclude/a.pyi
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_black.py b/tests/test_black.py
index e654d0d..c10dd1d 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -11,6 +11,7 @@
 from typing import Any, List, Tuple, Iterator
 import unittest
 from unittest.mock import patch
+import re
 
 from click import unstyle
 from click.testing import CliRunner
@@ -851,6 +852,53 @@ def test_pipe_force_py36(self) -> None:
         actual = result.output
         self.assertFormatEqual(actual, expected)
 
+    def test_include_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"/exclude/|/\.definitely_exclude/")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+        ]
+        sources.extend(black.gen_python_files_in_dir(path, include, exclude))
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_empty_include(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, empty, re.compile(black.DEFAULT_EXCLUDES)
+            )
+        )
+        self.assertEqual([], (sources))
+
+    def test_empty_exclude(self) -> None:
+        path = THIS_DIR / "include_exclude_tests"
+        empty = re.compile(r"")
+        sources: List[Path] = []
+        expected = [
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/dont_exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/exclude/a.pyi"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.py"),
+            Path(THIS_DIR / "include_exclude_tests/b/.definitely_exclude/a.pyi"),
+        ]
+        sources.extend(
+            black.gen_python_files_in_dir(
+                path, re.compile(black.DEFAULT_INCLUDES), empty
+            )
+        )
+        self.assertEqual(sorted(expected), sorted(sources))
+
+    def test_invalid_include_exclude(self) -> None:
+        for option in ["--include", "--exclude"]:
+            result = CliRunner().invoke(black.main, ["-", option, "**()(!!*)"])
+            self.assertEqual(result.exit_code, 2)
+
 
 if __name__ == "__main__":
     unittest.main()