Repository: psf/black

Issue:
cache significantly slows down black due to pathlib
**Describe the bug** A clear and concise description of what the bug is.

**To Reproduce** Steps to reproduce the behavior:

With a sufficiently large number of runs of black, the cache accumulates pretty rapidly and causes *significant* slowdown compared to uncached runs:

**uncached**
```console
$ rm -rf t; time XDG_CACHE_HOME=$PWD/t black t.py
reformatted t.py
All done! ✨ 🍰 ✨
1 file reformatted.

real	0m0.295s
user	0m0.237s
sys	0m0.036s
```

**cached**
```console
$ time black t.py
All done! ✨ 🍰 ✨
1 file left unchanged.

real	0m2.115s
user	0m1.999s
sys	0m0.089s
```

The cache in question isn't ~that large:

```console
$ du -hs ~/.cache/black/20.8b1/*.pickle
3.8M	/home/asottile/.cache/black/20.8b1/cache.-.88.1.0.pickle
16K	/home/asottile/.cache/black/20.8b1/Grammar3.8.5.final.0.pickle
4.0K	/home/asottile/.cache/black/20.8b1/PatternGrammar3.8.5.final.0.pickle
```

The reason for all the slowdown appears to be because pathlib is slow -- I've attached a pstats and svg of this:

![profile graph](https://user-images.githubusercontent.com/1810591/106480237-02b53480-6460-11eb-9165-b908c1de64da.png)

This was generated using [this procedure](https://youtu.be/ey_P64E34g0?list=PLWBKAf81pmOaP9naRiNAqug6EBnkPakvY)

```bash
python -m cProfile -o out.pstats -m black t.py
gprof2dot -z __init__:6604:patched_main out.pstats | dot -Tsvg -o out.svg  # gprof2dot from `yelp-gprof2dot`, `dot` from graphviz
```

[out.zip](https://github.com/psf/black/files/5904859/out.zip) (contains svg and pstats file)

**Expected behavior** A clear and concise description of what you expected to happen.

The cache shouldn't make the execution significantly slower

**Environment (please complete the following information):**

- Version: [e.g. master] 20.8b1
- OS and Python version: [e.g. Linux/Python 3.7.4rc1] 3.8.5 on ubuntu 20.04

**Does this bug also happen on master?** To answer this, you have two options:

yes, if I build up a new cache using the current default branch it reproduces as well

**Additional context** Add any other context about the problem here.

Removing pathlib from the Cache makes the cache serialization cost ~essentially zero -- will follow up with a demonstration patch



Discussion:
here is a patch removing pathlib from the Cache

```diff
$ git diff --staged | cat
diff --git a/src/black/__init__.py b/src/black/__init__.py
index 9034bf6..e5736f0 100644
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -93,7 +93,7 @@ Transformer = Callable[["Line", Collection["Feature"]], Iterator["Line"]]
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
-Cache = Dict[Path, CacheInfo]
+Cache = Dict[str, CacheInfo]
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
@@ -724,7 +724,8 @@ def reformat_one(
             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
                 cache = read_cache(mode)
                 res_src = src.resolve()
-                if res_src in cache and cache[res_src] == get_cache_info(res_src):
+                res_src_s = str(res_src)
+                if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
                     changed = Changed.CACHED
             if changed is not Changed.CACHED and format_file_in_place(
                 src, fast=fast, write_back=write_back, mode=mode
@@ -6781,8 +6782,8 @@ def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set
     """
     todo, done = set(), set()
     for src in sources:
-        src = src.resolve()
-        if cache.get(src) != get_cache_info(src):
+        res_src = src.resolve()
+        if cache.get(str(res_src)) != get_cache_info(res_src):
             todo.add(src)
         else:
             done.add(src)
@@ -6794,7 +6795,7 @@ def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None:
     cache_file = get_cache_file(mode)
     try:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
-        new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
+        new_cache = {**cache, **{str(src.resolve()): get_cache_info(src) for src in sources}}
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
             pickle.dump(new_cache, f, protocol=4)
         os.replace(f.name, cache_file)
```

and the associated profile from this

![out2](https://user-images.githubusercontent.com/1810591/106484965-fa132d00-6464-11eb-80a6-1592d7fe372a.png)

[out2.zip](https://github.com/psf/black/files/5905077/out2.zip)

I see, so the issue is that we pickle Path objects instead of strings in the pickled cache? That does seem like useless overhead; I'd merge a PR making that change.
++ this would be a big help, thanks @asottile! We recently discovered that our black runs were very slow under Jenkins due to a ~200 MB cache pickle:

```
>>> contents = pickle.load(open("19.10b0/cache.2,6.131.1.0.pickle", "rb"))

>>> type(contents)
<class 'dict'>

>>> len(contents)
1777951

>>> pprint(list(contents.items())[:5])
[(PosixPath('/ephemeral/jenkins/workspace/xxxxxxxx-52804/aaaaaaaa.py'),
  (1611104503.533281, 0)),
 (PosixPath('/ephemeral/jenkins/workspace/xxxxxxxx-52804/bbbbbbbb.py'),
  (1611104503.537281, 0)),
 (PosixPath('/ephemeral/jenkins/workspace/xxxxxxxx-52804/cccccccc.py'),
  (1611104503.4972813, 0)),
 (PosixPath('/ephemeral/jenkins/workspace/xxxxxxxx-52804/dddddddd.py'),
  (1611104503.557281, 0)),
 (PosixPath('/ephemeral/jenkins/workspace/xxxxxxxx-52804/eeeeeeee.py'),
  (1611104503.5212812, 576))]
```

Running black on a single file was taking 30 seconds until we cleared this cache (70+ minutes total when run using pre-commit on a large repo, compared to ~2 minutes after the cache was removed).
@chriskuehl some related thoughts: The Black cache on my company's CI instances is <1 MB on a repo with millions of lines, so I was surprised to see a 200 MB cache. I guess it's because the `xxxxxxxx` is some uuid-like string and every Jenkins run generates new cache entries for you, and your cache will essentially grow forever with every run. Storing strings instead of Path objects will therefore *help*, but your cache is still going to keep linearly growing forever. Our CI setup doesn't have that issue because we always use the same paths, which is nice because it means that most Black runs in our CI mostly hit the cache.

You could fix the unbounded growth on your side by configuring Black to use a fresh cache for every Jenkins run, or by just periodically deleting the cache. I'm trying to think of something that could be done in Black itself, but not sure there is much we can do.
@JelleZijlstra you're right, it's because our workspace names are unique and generally only used once. We're running black without the cache now on CI to work around this (setting `XDG_CACHE_HOME` to a location inside the temporary workspace), but it was sort of counter-intuitive for us that a populated cache leads to slower runs rather than faster ones.

From our perspective, it would be nice if black cleaned up its cache once it reached a size where loading the cache takes 30+ seconds, but I'm not sure how easy that is to implement or if that will affect other use-cases.

Resolving patch:
diff --git a/src/black/__init__.py b/src/black/__init__.py
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -93,7 +93,7 @@
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
-Cache = Dict[Path, CacheInfo]
+Cache = Dict[str, CacheInfo]
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
@@ -724,7 +724,8 @@ def reformat_one(
             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
                 cache = read_cache(mode)
                 res_src = src.resolve()
-                if res_src in cache and cache[res_src] == get_cache_info(res_src):
+                res_src_s = str(res_src)
+                if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
                     changed = Changed.CACHED
             if changed is not Changed.CACHED and format_file_in_place(
                 src, fast=fast, write_back=write_back, mode=mode
@@ -6781,8 +6782,8 @@ def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set
     """
     todo, done = set(), set()
     for src in sources:
-        src = src.resolve()
-        if cache.get(src) != get_cache_info(src):
+        res_src = src.resolve()
+        if cache.get(str(res_src)) != get_cache_info(res_src):
             todo.add(src)
         else:
             done.add(src)
@@ -6794,7 +6795,10 @@ def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None:
     cache_file = get_cache_file(mode)
     try:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
-        new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
+        new_cache = {
+            **cache,
+            **{str(src.resolve()): get_cache_info(src) for src in sources},
+        }
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
             pickle.dump(new_cache, f, protocol=4)
         os.replace(f.name, cache_file)


Test patch:
diff --git a/tests/test_black.py b/tests/test_black.py
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -1003,7 +1003,7 @@ def test_cache_broken_file(self) -> None:
                 fobj.write("print('hello')")
             self.invokeBlack([str(src)])
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
+            self.assertIn(str(src), cache)
 
     def test_cache_single_file_already_cached(self) -> None:
         mode = DEFAULT_MODE
@@ -1035,8 +1035,8 @@ def test_cache_multiple_files(self) -> None:
             with two.open("r") as fobj:
                 self.assertEqual(fobj.read(), 'print("hello")\n')
             cache = black.read_cache(mode)
-            self.assertIn(one, cache)
-            self.assertIn(two, cache)
+            self.assertIn(str(one), cache)
+            self.assertIn(str(two), cache)
 
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = DEFAULT_MODE
@@ -1116,8 +1116,8 @@ def test_write_cache_read_cache(self) -> None:
             src.touch()
             black.write_cache({}, [src], mode)
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
-            self.assertEqual(cache[src], black.get_cache_info(src))
+            self.assertIn(str(src), cache)
+            self.assertEqual(cache[str(src)], black.get_cache_info(src))
 
     def test_filter_cached(self) -> None:
         with TemporaryDirectory() as workspace:
@@ -1128,7 +1128,10 @@ def test_filter_cached(self) -> None:
             uncached.touch()
             cached.touch()
             cached_but_changed.touch()
-            cache = {cached: black.get_cache_info(cached), cached_but_changed: (0.0, 0)}
+            cache = {
+                str(cached): black.get_cache_info(cached),
+                str(cached_but_changed): (0.0, 0),
+            }
             todo, done = black.filter_cached(
                 cache, {uncached, cached, cached_but_changed}
             )
@@ -1156,8 +1159,8 @@ def test_failed_formatting_does_not_get_cached(self) -> None:
                 fobj.write('print("hello")\n')
             self.invokeBlack([str(workspace)], exit_code=123)
             cache = black.read_cache(mode)
-            self.assertNotIn(failing, cache)
-            self.assertIn(clean, cache)
+            self.assertNotIn(str(failing), cache)
+            self.assertIn(str(clean), cache)
 
     def test_write_cache_write_fail(self) -> None:
         mode = DEFAULT_MODE
@@ -1210,9 +1213,9 @@ def test_read_cache_line_lengths(self) -> None:
             path.touch()
             black.write_cache({}, [path], mode)
             one = black.read_cache(mode)
-            self.assertIn(path, one)
+            self.assertIn(str(path), one)
             two = black.read_cache(short_mode)
-            self.assertNotIn(path, two)
+            self.assertNotIn(str(path), two)
 
     def test_single_file_force_pyi(self) -> None:
         pyi_mode = replace(DEFAULT_MODE, is_pyi=True)
@@ -1226,9 +1229,9 @@ def test_single_file_force_pyi(self) -> None:
                 actual = fh.read()
             # verify cache with --pyi is separate
             pyi_cache = black.read_cache(pyi_mode)
-            self.assertIn(path, pyi_cache)
+            self.assertIn(str(path), pyi_cache)
             normal_cache = black.read_cache(DEFAULT_MODE)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(contents, actual)
         black.assert_stable(contents, actual, pyi_mode)
@@ -1255,8 +1258,8 @@ def test_multi_file_force_pyi(self) -> None:
             pyi_cache = black.read_cache(pyi_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_pyi(self) -> None:
         source, expected = read_data("force_pyi")
@@ -1280,9 +1283,9 @@ def test_single_file_force_py36(self) -> None:
                 actual = fh.read()
             # verify cache with --target-version is separate
             py36_cache = black.read_cache(py36_mode)
-            self.assertIn(path, py36_cache)
+            self.assertIn(str(path), py36_cache)
             normal_cache = black.read_cache(reg_mode)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertEqual(actual, expected)
 
     @event_loop()
@@ -1307,8 +1310,8 @@ def test_multi_file_force_py36(self) -> None:
             pyi_cache = black.read_cache(py36_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_py36(self) -> None:
         source, expected = read_data("force_py36")