Repository: psf/black

Issue:
cache significantly slows down black due to pathlib
**Describe the bug** A clear and concise description of what the bug is.

**To Reproduce** Steps to reproduce the behavior:

With a sufficiently large number of runs of black, the cache accumulates pretty rapidly and causes *significant* slowdown compared to uncached runs:

**uncached**
```console
$ rm -rf t; time XDG_CACHE_HOME=$PWD/t black t.py
reformatted t.py
All done! ✨ 🍰 ✨
1 file reformatted.

real	0m0.295s
user	0m0.237s
sys	0m0.036s
```

**cached**
```console
$ time black t.py
All done! ✨ 🍰 ✨
1 file left unchanged.

real	0m2.115s
user	0m1.999s
sys	0m0.089s
```

The cache in question isn't ~that large:

```console
$ du -hs ~/.cache/black/20.8b1/*.pickle
3.8M	/home/asottile/.cache/black/20.8b1/cache.-.88.1.0.pickle
16K	/home/asottile/.cache/black/20.8b1/Grammar3.8.5.final.0.pickle
4.0K	/home/asottile/.cache/black/20.8b1/PatternGrammar3.8.5.final.0.pickle
```

The reason for all the slowdown appears to be because pathlib is slow -- I've attached a pstats and svg of this:

![profile graph](https://user-images.githubusercontent.com/1810591/106480237-02b53480-6460-11eb-9165-b908c1de64da.png)

This was generated using [this procedure](https://youtu.be/ey_P64E34g0?list=PLWBKAf81pmOaP9naRiNAqug6EBnkPakvY)

```bash
python -m cProfile -o out.pstats -m black t.py
gprof2dot -z __init__:6604:patched_main out.pstats | dot -Tsvg -o out.svg  # gprof2dot from `yelp-gprof2dot`, `dot` from graphviz
```

[out.zip](https://github.com/psf/black/files/5904859/out.zip) (contains svg and pstats file)

**Expected behavior** A clear and concise description of what you expected to happen.

The cache shouldn't make the execution significantly slower

**Environment (please complete the following information):**

- Version: [e.g. master] 20.8b1
- OS and Python version: [e.g. Linux/Python 3.7.4rc1] 3.8.5 on ubuntu 20.04

**Does this bug also happen on master?** To answer this, you have two options:

yes, if I build up a new cache using the current default branch it reproduces as well

**Additional context** Add any other context about the problem here.

Removing pathlib from the Cache makes the cache serialization cost ~essentially zero -- will follow up with a demonstration patch


Discussion:
JelleZijlstra: Looks like Travis is just never finishing, do we still need it?
JelleZijlstra: And thanks for catching and fixing this!
ichard26: > Looks like Travis is just never finishing, do we still need it?

Travis CI for Open Source is shutting down in a few weeks. I'm 99.99% sure we don't need it as our Test, Lint, Docs, Upload / Package, Primer, and Fuzz workflows are all on GitHub Actions. So even though we *can* migrate to the .com version with its 1000 free Linux minutes(?), I don't think we need to.

([more info here](https://blog.travis-ci.com/oss-announcement), [here](https://blog.travis-ci.com/2020-11-02-travis-ci-new-billing), and [also here](https://docs.travis-ci.com/user/migrate/open-source-repository-migration))
asottile: added the changelog entry

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index ca8a047..5f7ca4f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -26,6 +26,8 @@
 
 - use lowercase hex strings (#1692)
 
+- speed up caching by avoiding pathlib (#1950)
+
 #### _Packaging_
 
 - Self-contained native _Black_ binaries are now provided for releases via GitHub
diff --git a/src/black/__init__.py b/src/black/__init__.py
index 9034bf6..7c1a013 100644
--- a/src/black/__init__.py
+++ b/src/black/__init__.py
@@ -93,7 +93,7 @@
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
-Cache = Dict[Path, CacheInfo]
+Cache = Dict[str, CacheInfo]
 out = partial(click.secho, bold=True, err=True)
 err = partial(click.secho, fg="red", err=True)
 
@@ -724,7 +724,8 @@ def reformat_one(
             if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):
                 cache = read_cache(mode)
                 res_src = src.resolve()
-                if res_src in cache and cache[res_src] == get_cache_info(res_src):
+                res_src_s = str(res_src)
+                if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src):
                     changed = Changed.CACHED
             if changed is not Changed.CACHED and format_file_in_place(
                 src, fast=fast, write_back=write_back, mode=mode
@@ -6781,8 +6782,8 @@ def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set
     """
     todo, done = set(), set()
     for src in sources:
-        src = src.resolve()
-        if cache.get(src) != get_cache_info(src):
+        res_src = src.resolve()
+        if cache.get(str(res_src)) != get_cache_info(res_src):
             todo.add(src)
         else:
             done.add(src)
@@ -6794,7 +6795,10 @@ def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None:
     cache_file = get_cache_file(mode)
     try:
         CACHE_DIR.mkdir(parents=True, exist_ok=True)
-        new_cache = {**cache, **{src.resolve(): get_cache_info(src) for src in sources}}
+        new_cache = {
+            **cache,
+            **{str(src.resolve()): get_cache_info(src) for src in sources},
+        }
         with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f:
             pickle.dump(new_cache, f, protocol=4)
         os.replace(f.name, cache_file)


Test patch:
diff --git a/tests/test_black.py b/tests/test_black.py
index 28b7578..cfd3cbd 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -1003,7 +1003,7 @@ def test_cache_broken_file(self) -> None:
                 fobj.write("print('hello')")
             self.invokeBlack([str(src)])
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
+            self.assertIn(str(src), cache)
 
     def test_cache_single_file_already_cached(self) -> None:
         mode = DEFAULT_MODE
@@ -1035,8 +1035,8 @@ def test_cache_multiple_files(self) -> None:
             with two.open("r") as fobj:
                 self.assertEqual(fobj.read(), 'print("hello")\n')
             cache = black.read_cache(mode)
-            self.assertIn(one, cache)
-            self.assertIn(two, cache)
+            self.assertIn(str(one), cache)
+            self.assertIn(str(two), cache)
 
     def test_no_cache_when_writeback_diff(self) -> None:
         mode = DEFAULT_MODE
@@ -1116,8 +1116,8 @@ def test_write_cache_read_cache(self) -> None:
             src.touch()
             black.write_cache({}, [src], mode)
             cache = black.read_cache(mode)
-            self.assertIn(src, cache)
-            self.assertEqual(cache[src], black.get_cache_info(src))
+            self.assertIn(str(src), cache)
+            self.assertEqual(cache[str(src)], black.get_cache_info(src))
 
     def test_filter_cached(self) -> None:
         with TemporaryDirectory() as workspace:
@@ -1128,7 +1128,10 @@ def test_filter_cached(self) -> None:
             uncached.touch()
             cached.touch()
             cached_but_changed.touch()
-            cache = {cached: black.get_cache_info(cached), cached_but_changed: (0.0, 0)}
+            cache = {
+                str(cached): black.get_cache_info(cached),
+                str(cached_but_changed): (0.0, 0),
+            }
             todo, done = black.filter_cached(
                 cache, {uncached, cached, cached_but_changed}
             )
@@ -1156,8 +1159,8 @@ def test_failed_formatting_does_not_get_cached(self) -> None:
                 fobj.write('print("hello")\n')
             self.invokeBlack([str(workspace)], exit_code=123)
             cache = black.read_cache(mode)
-            self.assertNotIn(failing, cache)
-            self.assertIn(clean, cache)
+            self.assertNotIn(str(failing), cache)
+            self.assertIn(str(clean), cache)
 
     def test_write_cache_write_fail(self) -> None:
         mode = DEFAULT_MODE
@@ -1210,9 +1213,9 @@ def test_read_cache_line_lengths(self) -> None:
             path.touch()
             black.write_cache({}, [path], mode)
             one = black.read_cache(mode)
-            self.assertIn(path, one)
+            self.assertIn(str(path), one)
             two = black.read_cache(short_mode)
-            self.assertNotIn(path, two)
+            self.assertNotIn(str(path), two)
 
     def test_single_file_force_pyi(self) -> None:
         pyi_mode = replace(DEFAULT_MODE, is_pyi=True)
@@ -1226,9 +1229,9 @@ def test_single_file_force_pyi(self) -> None:
                 actual = fh.read()
             # verify cache with --pyi is separate
             pyi_cache = black.read_cache(pyi_mode)
-            self.assertIn(path, pyi_cache)
+            self.assertIn(str(path), pyi_cache)
             normal_cache = black.read_cache(DEFAULT_MODE)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertFormatEqual(expected, actual)
         black.assert_equivalent(contents, actual)
         black.assert_stable(contents, actual, pyi_mode)
@@ -1255,8 +1258,8 @@ def test_multi_file_force_pyi(self) -> None:
             pyi_cache = black.read_cache(pyi_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_pyi(self) -> None:
         source, expected = read_data("force_pyi")
@@ -1280,9 +1283,9 @@ def test_single_file_force_py36(self) -> None:
                 actual = fh.read()
             # verify cache with --target-version is separate
             py36_cache = black.read_cache(py36_mode)
-            self.assertIn(path, py36_cache)
+            self.assertIn(str(path), py36_cache)
             normal_cache = black.read_cache(reg_mode)
-            self.assertNotIn(path, normal_cache)
+            self.assertNotIn(str(path), normal_cache)
         self.assertEqual(actual, expected)
 
     @event_loop()
@@ -1307,8 +1310,8 @@ def test_multi_file_force_py36(self) -> None:
             pyi_cache = black.read_cache(py36_mode)
             normal_cache = black.read_cache(reg_mode)
             for path in paths:
-                self.assertIn(path, pyi_cache)
-                self.assertNotIn(path, normal_cache)
+                self.assertIn(str(path), pyi_cache)
+                self.assertNotIn(str(path), normal_cache)
 
     def test_pipe_force_py36(self) -> None:
         source, expected = read_data("force_py36")