Repository: dask/dask
Pull request: Deprecate `npartitions='auto'` for set_index & sort_values
- [x] Closes #10736 
- [x] Tests added / passed
- [x] Passes `pre-commit run --all-files`


Problem statement:
Deprecate ``npartitions="auto"`` for set_index and sort_values
The docs state that this makes a decision based on memory use

> If ‘auto’ then decide by memory use.

But that's just wrong

```
    if npartitions == "auto":
        repartition = True
        npartitions = max(100, df.npartitions)
```

This is very prohibitive for larger workloads, imagine a DataFrame with thousands of partitions getting compressed that heavily, so let's just get rid of this option

Hints from issue discussion:

Commit message: Deprecate `npartitions='auto'` for set_index & sort_values

Patch to review:
diff --git a/dask/dataframe/shuffle.py b/dask/dataframe/shuffle.py
index 270df6da4e1..87fc85f93b2 100644
--- a/dask/dataframe/shuffle.py
+++ b/dask/dataframe/shuffle.py
@@ -166,6 +166,7 @@ def sort_values(
         return df.map_partitions(sort_function, **sort_kwargs)
 
     if npartitions == "auto":
+        warnings.warn("`npartitions='auto'` is deprecated.", FutureWarning, 2)
         repartition = True
         npartitions = max(100, df.npartitions)
     else:
@@ -235,6 +236,7 @@ def set_index(
         ).clear_divisions()
 
     if npartitions == "auto":
+        warnings.warn("`npartitions='auto'` is deprecated.", FutureWarning, 2)
         repartition = True
         npartitions = max(100, df.npartitions)
     else:
diff --git a/dask/dataframe/tests/test_shuffle.py b/dask/dataframe/tests/test_shuffle.py
index cd3e173d1de..cabfac240b5 100644
--- a/dask/dataframe/tests/test_shuffle.py
+++ b/dask/dataframe/tests/test_shuffle.py
@@ -569,19 +569,19 @@ def _set_index(i, df, idx):
     return df.set_index(idx).divisions
 
 
-def test_set_index_reduces_partitions_small(shuffle_method):
+def test_set_index_not_reduce_partitions_small(shuffle_method):
     df = pd.DataFrame({"x": np.random.random(100)})
     ddf = dd.from_pandas(df, npartitions=50)
 
-    ddf2 = ddf.set_index("x", shuffle_method=shuffle_method, npartitions="auto")
-    assert ddf2.npartitions < 10
+    ddf2 = ddf.set_index("x", shuffle_method=shuffle_method)
+    assert ddf2.npartitions == ddf.npartitions
 
 
 def make_part(n):
     return pd.DataFrame({"x": np.random.random(n), "y": np.random.random(n)})
 
 
-def test_set_index_reduces_partitions_large(shuffle_method):
+def test_set_index_not_reduce_partitions_large(shuffle_method):
     nbytes = 1e6
     nparts = 50
     n = int(nbytes / (nparts * 8))
@@ -591,10 +591,8 @@ def test_set_index_reduces_partitions_large(shuffle_method):
         make_part(1),
         [None] * (nparts + 1),
     )
-    ddf2 = ddf.set_index(
-        "x", shuffle_method=shuffle_method, npartitions="auto", partition_size=nbytes
-    )
-    assert 1 < ddf2.npartitions < 20
+    ddf2 = ddf.set_index("x", shuffle_method=shuffle_method, partition_size=nbytes)
+    assert ddf2.npartitions == ddf.npartitions
 
 
 def test_set_index_doesnt_increase_partitions(shuffle_method):
@@ -607,9 +605,7 @@ def test_set_index_doesnt_increase_partitions(shuffle_method):
         make_part(1),
         [None] * (nparts + 1),
     )
-    ddf2 = ddf.set_index(
-        "x", shuffle_method=shuffle_method, npartitions="auto", partition_size=nbytes
-    )
+    ddf2 = ddf.set_index("x", shuffle_method=shuffle_method, partition_size=nbytes)
     assert ddf2.npartitions <= ddf.npartitions
 
 
@@ -1107,8 +1103,7 @@ def test_gh_2730():
     tm.assert_frame_equal(result.sort_values("KEY").reset_index(drop=True), expected)
 
 
-@pytest.mark.parametrize("npartitions", [None, "auto"])
-def test_set_index_does_not_repeat_work_due_to_optimizations(npartitions):
+def test_set_index_does_not_repeat_work_due_to_optimizations():
     # Atomic counter
     count = itertools.count()
 
@@ -1126,7 +1121,7 @@ def make_part(dummy, n):
     dsk.update({("x", i): (make_part, ("inc", i), n) for i in range(nparts)})
     ddf = dd.DataFrame(dsk, "x", make_part(None, 1), [None] * (nparts + 1))
 
-    ddf.set_index("x", npartitions=npartitions)
+    ddf.set_index("x")
     ntimes = next(count)
     assert ntimes == nparts
 
@@ -1628,7 +1623,7 @@ def test_shuffle_by_as_list():
     df = pd.DataFrame({"a": [1, 3, 2]})
     ddf = dd.from_pandas(df, npartitions=3)
     with dask.config.set(scheduler="single-threaded"):
-        got = ddf.sort_values(by=["a"], npartitions="auto", ascending=True)
+        got = ddf.sort_values(by=["a"], ascending=True)
     expect = pd.DataFrame({"a": [1, 2, 3]})
     dd.assert_eq(got, expect, check_index=False)
 


Code review comments:
diff_hunk: @@ -569,19 +569,19 @@ def _set_index(i, df, idx):
     return df.set_index(idx).divisions
 
 
-def test_set_index_reduces_partitions_small(shuffle_method):
+def test_set_index_not_reduce_partitions_small(shuffle_method):
path: dask/dataframe/tests/test_shuffle.py
text: @user1:
I think you can just rip out this test