Repository: pandas-dev/pandas
Pull request: ENH: Add engine keyword to expanding.apply to utilize Numba
- [x] closes #30936
- [x] tests added / passed
- [x] passes `black pandas`
- [x] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
- [x] whatsnew entry


Problem statement:
ENH: Add engine keyword argument to expanding.apply to leverage Numba
In 1.0, we're adding a `engine` keyword argument to `rolling.apply` to run the routine using Cython or Numba.

By natural extension, we should allow `expanding.apply` to utilize Numba by adding the same `engine` keyword argument to `expanding.apply`

Hints from issue discussion:

Commit message: Add engine keyword to expanding.apply to utilize Numba

Patch to review:
diff --git a/doc/source/user_guide/computation.rst b/doc/source/user_guide/computation.rst
index a2150c207c0b0..aeb32db639ffb 100644
--- a/doc/source/user_guide/computation.rst
+++ b/doc/source/user_guide/computation.rst
@@ -348,6 +348,7 @@ Numba will be applied in potentially two routines:
 
 1. If ``func`` is a standard Python function, the engine will `JIT <http://numba.pydata.org/numba-doc/latest/user/overview.html>`__
 the passed function. ``func`` can also be a JITed function in which case the engine will not JIT the function again.
+
 2. The engine will JIT the for loop where the apply function is applied to each window.
 
 The ``engine_kwargs`` argument is a dictionary of keyword arguments that will be passed into the
diff --git a/pandas/core/window/expanding.py b/pandas/core/window/expanding.py
index 68c3514308cbc..279af82b2bad3 100644
--- a/pandas/core/window/expanding.py
+++ b/pandas/core/window/expanding.py
@@ -148,8 +148,23 @@ def count(self, **kwargs):
 
     @Substitution(name="expanding")
     @Appender(_shared_docs["apply"])
-    def apply(self, func, raw=False, args=(), kwargs={}):
-        return super().apply(func, raw=raw, args=args, kwargs=kwargs)
+    def apply(
+        self,
+        func,
+        raw=False,
+        engine="cython",
+        engine_kwargs=None,
+        args=None,
+        kwargs=None,
+    ):
+        return super().apply(
+            func,
+            raw=raw,
+            engine=engine,
+            engine_kwargs=engine_kwargs,
+            args=args,
+            kwargs=kwargs,
+        )
 
     @Substitution(name="expanding")
     @Appender(_shared_docs["sum"])
diff --git a/pandas/core/window/rolling.py b/pandas/core/window/rolling.py
index f612826132fd7..2b327ec406923 100644
--- a/pandas/core/window/rolling.py
+++ b/pandas/core/window/rolling.py
@@ -1203,7 +1203,7 @@ def count(self):
 
     _shared_docs["apply"] = dedent(
         r"""
-    The %(name)s function's apply function.
+    Apply an arbitrary function to each %(name)s window.
 
     Parameters
     ----------
diff --git a/pandas/tests/window/moments/test_moments_expanding.py b/pandas/tests/window/moments/test_moments_expanding.py
index 4596552d8f255..322082187f531 100644
--- a/pandas/tests/window/moments/test_moments_expanding.py
+++ b/pandas/tests/window/moments/test_moments_expanding.py
@@ -13,15 +13,17 @@ class TestExpandingMomentsConsistency(ConsistencyBase):
     def setup_method(self, method):
         self._create_data()
 
-    def test_expanding_apply_args_kwargs(self, raw):
+    def test_expanding_apply_args_kwargs(self, engine_and_raw):
         def mean_w_arg(x, const):
             return np.mean(x) + const
 
+        engine, raw = engine_and_raw
+
         df = DataFrame(np.random.rand(20, 3))
 
-        expected = df.expanding().apply(np.mean, raw=raw) + 20.0
+        expected = df.expanding().apply(np.mean, engine=engine, raw=raw) + 20.0
 
-        result = df.expanding().apply(mean_w_arg, raw=raw, args=(20,))
+        result = df.expanding().apply(mean_w_arg, engine=engine, raw=raw, args=(20,))
         tm.assert_frame_equal(result, expected)
 
         result = df.expanding().apply(mean_w_arg, raw=raw, kwargs={"const": 20})
@@ -190,11 +192,14 @@ def expanding_func(x, min_periods=1, center=False, axis=0):
         )
 
     @pytest.mark.parametrize("has_min_periods", [True, False])
-    def test_expanding_apply(self, raw, has_min_periods):
+    def test_expanding_apply(self, engine_and_raw, has_min_periods):
+
+        engine, raw = engine_and_raw
+
         def expanding_mean(x, min_periods=1):
 
             exp = x.expanding(min_periods=min_periods)
-            result = exp.apply(lambda x: x.mean(), raw=raw)
+            result = exp.apply(lambda x: x.mean(), raw=raw, engine=engine)
             return result
 
         # TODO(jreback), needed to add preserve_nan=False
@@ -202,14 +207,20 @@ def expanding_mean(x, min_periods=1):
         self._check_expanding(expanding_mean, np.mean, preserve_nan=False)
         self._check_expanding_has_min_periods(expanding_mean, np.mean, has_min_periods)
 
-    def test_expanding_apply_empty_series(self, raw):
+    def test_expanding_apply_empty_series(self, engine_and_raw):
+        engine, raw = engine_and_raw
         ser = Series([], dtype=np.float64)
-        tm.assert_series_equal(ser, ser.expanding().apply(lambda x: x.mean(), raw=raw))
+        tm.assert_series_equal(
+            ser, ser.expanding().apply(lambda x: x.mean(), raw=raw, engine=engine)
+        )
 
-    def test_expanding_apply_min_periods_0(self, raw):
+    def test_expanding_apply_min_periods_0(self, engine_and_raw):
         # GH 8080
+        engine, raw = engine_and_raw
         s = Series([None, None, None])
-        result = s.expanding(min_periods=0).apply(lambda x: len(x), raw=raw)
+        result = s.expanding(min_periods=0).apply(
+            lambda x: len(x), raw=raw, engine=engine
+        )
         expected = Series([1.0, 2.0, 3.0])
         tm.assert_series_equal(result, expected)
 


Code review comments:
diff_hunk: @@ -148,8 +148,23 @@ def count(self, **kwargs):
 
     @Substitution(name="expanding")
     @Appender(_shared_docs["apply"])
-    def apply(self, func, raw=False, args=(), kwargs={}):
-        return super().apply(func, raw=raw, args=args, kwargs=kwargs)
+    def apply(
+        self,
+        func,
+        raw=False,
+        engine="cython",
+        engine_kwargs=None,
path: pandas/core/window/expanding.py
text: @user1:
Can you add annotation for this? I assume `Dict[str, Any]` unless can further subset the provided values

diff_hunk: @@ -148,8 +148,23 @@ def count(self, **kwargs):
 
     @Substitution(name="expanding")
     @Appender(_shared_docs["apply"])
-    def apply(self, func, raw=False, args=(), kwargs={}):
-        return super().apply(func, raw=raw, args=args, kwargs=kwargs)
+    def apply(
+        self,
+        func,
+        raw=False,
+        engine="cython",
+        engine_kwargs=None,
+        args=None,
path: pandas/core/window/expanding.py
text: @user1:
Is this totally equivalent? I get the impression previously should have been *args and **kwargs so a little strange as is, but I don't think converting to `None` is totally the same?

@author:
Previously the docs described `rolling/expanding.apply` being able to take *args, **kwargs: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.core.window.Expanding.apply.html

But in actuality, the arguments were `args=(), kwargs={}`: https://github.com/pandas-dev/pandas/blob/0.25.x/pandas/core/window.py#L2128

Since the docs were incorrect, and the prior arguments has a mutable default argument, I changed `rolling.apply` to be  `args=None, kwargs=None` (later converting to `args=(), kwargs={}` internally if still `None`), and I am doing the same here for `expanding.apply`.

So _slightly_ changing the behavior, but taking advantage of the fact the docs were incorrect in the first place.