Repository: dask/dask

Issue:
Fix  `DataFrame.dropna` match pandas upstream bug fix
Currently, these  two UPSTREAM test are broken:

- [test_map_partitions_with_delayed_collection](https://github.com/dask/dask/runs/7331385905?check_suite_focus=true#step:5:11853)
- [test_dropna](https://github.com/dask/dask/runs/7331385905?check_suite_focus=true#step:5:11560)

Due to a [BUG-fix upstream](https://github.com/ncclementi/pandas/commit/94280f021bda7fb23e26db6e485e5367f2de8963), that prevents providing `how` and `thresh` simultaneously to `pd.DataFrame.dropna()` . The change involves modifying the default values `how='any', thresh=None,` to `how=_NoDefault.no_default, thresh=_NoDefault.no_default,` (see https://pandas.pydata.org/docs/dev/reference/api/pandas.DataFrame.dropna.html ) where` _NoDefault.no_default` comes from https://github.com/pandas-dev/pandas/blob/e7afa4b641b146874d17c36caa8a050bfde31283/pandas/_libs/lib.pyi#L27-L31 



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -5055,10 +5055,23 @@ def eval(self, expr, inplace=None, **kwargs):
         return self.map_partitions(M.eval, expr, meta=meta, inplace=inplace, **kwargs)
 
     @derived_from(pd.DataFrame)
-    def dropna(self, how="any", subset=None, thresh=None):
-        return self.map_partitions(
-            M.dropna, how=how, subset=subset, thresh=thresh, enforce_metadata=False
-        )
+    def dropna(self, how=no_default, subset=None, thresh=no_default):
+        # These keywords are incompatible with each other.
+        # Don't allow them both to be set.
+        if how is not no_default and thresh is not no_default:
+            raise TypeError(
+                "You cannot set both the how and thresh arguments at the same time."
+            )
+
+        # Only specify `how` or `thresh` keyword if specified by the user
+        # so we utilize other `dropna` keyword defaults appropriately
+        kwargs = {"subset": subset}
+        if how is not no_default:
+            kwargs["how"] = how
+        elif thresh is not no_default:
+            kwargs["thresh"] = thresh
+
+        return self.map_partitions(M.dropna, **kwargs, enforce_metadata=False)
 
     @derived_from(pd.DataFrame)
     def clip(self, lower=None, upper=None, out=None):


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -762,21 +762,17 @@ def test_dropna():
     )
 
     # threshold
-    assert_eq(df.dropna(thresh=None), df.loc[[20, 40]])
     assert_eq(ddf.dropna(thresh=None), df.dropna(thresh=None))
-
-    assert_eq(df.dropna(thresh=0), df.loc[:])
     assert_eq(ddf.dropna(thresh=0), df.dropna(thresh=0))
-
-    assert_eq(df.dropna(thresh=1), df.loc[[10, 20, 30, 40, 60]])
     assert_eq(ddf.dropna(thresh=1), df.dropna(thresh=1))
-
-    assert_eq(df.dropna(thresh=2), df.loc[[10, 20, 30, 40, 60]])
     assert_eq(ddf.dropna(thresh=2), df.dropna(thresh=2))
-
-    assert_eq(df.dropna(thresh=3), df.loc[[20, 40]])
     assert_eq(ddf.dropna(thresh=3), df.dropna(thresh=3))
 
+    # fail when how and thresh are both provided
+    # see https://github.com/dask/dask/issues/9365
+    with pytest.raises(TypeError, match="cannot set both the how and thresh arguments"):
+        ddf.dropna(how="all", thresh=0)
+
     # Regression test for https://github.com/dask/dask/issues/6540
     df = pd.DataFrame({"_0": [0, 0, np.nan], "_1": [1, 2, 3]})
     ddf = dd.from_pandas(df, npartitions=2)