Repository: dask/dask

Issue:
Add kurtosis method to dask dataframe
As the title said, I'd like to add the kurtosis-method to dask dataframes. Should be pretty much a copy of the skewness addition in #6881.


Discussion:

Resolving patch:
diff --git a/dask/array/stats.py b/dask/array/stats.py
--- a/dask/array/stats.py
+++ b/dask/array/stats.py
@@ -263,8 +263,11 @@ def kurtosis(a, axis=0, fisher=True, bias=True, nan_policy="propagate"):
     if fisher:
         return vals - 3
     else:
+        if vals.ndim == 0:
+            # TODO: scalar, min is a workaround
+            return vals.min()
+
         return vals
-        # TODO: scalar; vals = vals.item()  # array scalar
 
 
 @derived_from(scipy.stats)
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -2153,6 +2153,119 @@ def _skew_numeric(self, bias=True, nan_policy="propagate"):
             graph, name, num._meta_nonempty.skew(), divisions=[None, None]
         )
 
+    @derived_from(pd.DataFrame)
+    def kurtosis(
+        self, axis=None, fisher=True, bias=True, nan_policy="propagate", out=None
+    ):
+        """
+        .. note::
+
+           This implementation follows the dask.array.stats implementation
+           of kurtosis and calculates kurtosis without taking into account
+           a bias term for finite sample size, which corresponds to the
+           default settings of the scipy.stats kurtosis calculation. This differs
+           from pandas.
+
+           Further, this method currently does not support filtering out NaN
+           values, which is again a difference to Pandas.
+        """
+        axis = self._validate_axis(axis)
+        _raise_if_object_series(self, "kurtosis")
+        meta = self._meta_nonempty.kurtosis()
+        if axis == 1:
+            result = map_partitions(
+                M.kurtosis,
+                self,
+                meta=meta,
+                token=self._token_prefix + "kurtosis",
+                axis=axis,
+                enforce_metadata=False,
+            )
+            return handle_out(out, result)
+        else:
+            if self.ndim == 1:
+                result = self._kurtosis_1d(
+                    self, fisher=fisher, bias=bias, nan_policy=nan_policy
+                )
+                return handle_out(out, result)
+            else:
+                result = self._kurtosis_numeric(
+                    fisher=fisher, bias=bias, nan_policy=nan_policy
+                )
+
+            if isinstance(self, DataFrame):
+                result.divisions = (self.columns.min(), self.columns.max())
+
+            return handle_out(out, result)
+
+    def _kurtosis_1d(self, column, fisher=True, bias=True, nan_policy="propagate"):
+        """1D version of the kurtosis calculation.
+
+        Uses the array version from da.stats in case we are passing in a single series
+        """
+        # import depends on scipy, not installed by default
+        from ..array import stats as da_stats
+
+        if pd.api.types.is_integer_dtype(column._meta_nonempty):
+            column = column.astype("f8")
+
+        if not np.issubdtype(column.dtype, np.number):
+            column = column.astype("f8")
+
+        name = self._token_prefix + "kurtosis-1d-" + tokenize(column)
+
+        array_kurtosis = da_stats.kurtosis(
+            column.values, axis=0, fisher=fisher, bias=bias, nan_policy=nan_policy
+        )
+
+        layer = {
+            (name, 0): (methods.wrap_kurtosis_reduction, (array_kurtosis._name,), None)
+        }
+        graph = HighLevelGraph.from_collections(
+            name, layer, dependencies=[array_kurtosis]
+        )
+
+        return new_dd_object(
+            graph, name, column._meta_nonempty.kurtosis(), divisions=[None, None]
+        )
+
+    def _kurtosis_numeric(self, fisher=True, bias=True, nan_policy="propagate"):
+        """Method for dataframes with numeric columns.
+
+        Maps the array version from da.stats onto the numeric array of columns.
+        """
+        # import depends on scipy, not installed by default
+        from ..array import stats as da_stats
+
+        num = self.select_dtypes(include=["number", "bool"], exclude=[np.timedelta64])
+
+        values_dtype = num.values.dtype
+        array_values = num.values
+
+        if not np.issubdtype(values_dtype, np.number):
+            array_values = num.values.astype("f8")
+
+        array_kurtosis = da_stats.kurtosis(
+            array_values, axis=0, fisher=fisher, bias=bias, nan_policy=nan_policy
+        )
+
+        name = self._token_prefix + "kurtosis-numeric" + tokenize(num)
+        cols = num._meta.columns if is_dataframe_like(num) else None
+
+        kurtosis_shape = num._meta_nonempty.values.var(axis=0).shape
+        array_kurtosis_name = (array_kurtosis._name,) + (0,) * len(kurtosis_shape)
+
+        layer = {
+            (name, 0): (methods.wrap_kurtosis_reduction, array_kurtosis_name, cols)
+        }
+        graph = HighLevelGraph.from_collections(
+            name, layer, dependencies=[array_kurtosis]
+        )
+
+        return new_dd_object(
+            graph, name, num._meta_nonempty.kurtosis(), divisions=[None, None]
+        )
+
     @derived_from(pd.DataFrame)
     def sem(self, axis=None, skipna=None, ddof=1, split_every=False):
         axis = self._validate_axis(axis)
diff --git a/dask/dataframe/methods.py b/dask/dataframe/methods.py
--- a/dask/dataframe/methods.py
+++ b/dask/dataframe/methods.py
@@ -135,6 +135,13 @@ def wrap_skew_reduction(array_skew, index):
     return array_skew
 
 
+def wrap_kurtosis_reduction(array_kurtosis, index):
+    if isinstance(array_kurtosis, np.ndarray) or isinstance(array_kurtosis, list):
+        return pd.Series(array_kurtosis, index=index)
+
+    return array_kurtosis
+
+
 def var_mixed_concat(numeric_var, timedelta_var, columns):
     vars = pd.concat([numeric_var, timedelta_var])
 


Test patch:
diff --git a/dask/array/tests/test_stats.py b/dask/array/tests/test_stats.py
--- a/dask/array/tests/test_stats.py
+++ b/dask/array/tests/test_stats.py
@@ -151,3 +151,13 @@ def test_skew_single_return_type():
     dask_array = da.from_array(numpy_array, 3)
     result = dask.array.stats.skew(dask_array).compute()
     assert isinstance(result, np.float64)
+
+
+def test_kurtosis_single_return_type():
+    """This function tests the return type for the kurtosis method for a 1d array."""
+    numpy_array = np.random.random(size=(30,))
+    dask_array = da.from_array(numpy_array, 3)
+    result = dask.array.stats.kurtosis(dask_array).compute()
+    result_non_fisher = dask.array.stats.kurtosis(dask_array, fisher=False).compute()
+    assert isinstance(result, np.float64)
+    assert isinstance(result_non_fisher, np.float64)
diff --git a/dask/dataframe/tests/test_arithmetics_reduction.py b/dask/dataframe/tests/test_arithmetics_reduction.py
--- a/dask/dataframe/tests/test_arithmetics_reduction.py
+++ b/dask/dataframe/tests/test_arithmetics_reduction.py
@@ -732,6 +732,13 @@ def test_reductions(split_every):
             bias_factor = (n * (n - 1)) ** 0.5 / (n - 2)
             assert_eq(dds.skew(), pds.skew() / bias_factor)
 
+        if scipy:
+            # pandas uses a bias factor for kurtosis, need to correct for that
+            n = pds.shape[0]
+            factor = ((n - 1) * (n + 1)) / ((n - 2) * (n - 3))
+            offset = (6 * (n - 1)) / ((n - 2) * (n - 3))
+            assert_eq(factor * dds.kurtosis() + offset, pds.kurtosis())
+
         with pytest.warns(None):
             # runtime warnings; https://github.com/dask/dask/issues/2381
             assert_eq(dds.std(split_every=split_every), pds.std())
@@ -988,6 +995,7 @@ def check_raises(d, p, func):
     check_raises(dds, pds, "var")
     check_raises(dds, pds, "sem")
     check_raises(dds, pds, "skew")
+    check_raises(dds, pds, "kurtosis")
     assert_eq(dds.nunique(), pds.nunique())
 
     for pds in [
@@ -1011,6 +1019,7 @@ def check_raises(d, p, func):
         check_raises(dds, pds, "var")
         check_raises(dds, pds, "sem")
         check_raises(dds, pds, "skew")
+        check_raises(dds, pds, "kurtosis")
         assert_eq(dds.nunique(), pds.nunique())
 
     pds = pd.Series(pd.timedelta_range("1 days", freq="D", periods=5))
@@ -1021,6 +1030,7 @@ def check_raises(d, p, func):
     assert_eq(dds.count(), pds.count())
     # both pandas and dask skew calculations do not support timedelta
     check_raises(dds, pds, "skew")
+    check_raises(dds, pds, "kurtosis")
 
     # ToDo: pandas supports timedelta std, dask returns float64
     # assert_eq(dds.std(), pds.std())