Repository: scipy/scipy

Issue:
BUG: optimize.minimize: garbage collection in `lbfgs`
### Describe your issue.

Iterated calls to LBFGS increase memory consumption if garbage collection is not explicitly called.

This pipped up in https://github.com/scikit-learn/scikit-learn/issues/28993.

### Reproducing Code Example

```python
import tracemalloc

from scipy.optimize import minimize
import numpy as np


def do_something(*args, **kwargs):
    return np.random.rand()


def do_train():
    minimize(
        do_something,
        np.random.rand(300),
        args=(np.random.rand(2000, 3000)),
        method="L-BFGS-B"
    )


def run(reps):
    tracemalloc.start()
    for _ in range(reps):
        do_train()

    base, peak = tracemalloc.get_traced_memory()
    tracemalloc.stop()
    print(f"| {reps:^6} | {base/ 1024**2:^9.2f} | {peak/ 1024**2:^9.2f} | {(peak-base)/ 1024**2:^9.2f} |")


if __name__ == "__main__":
    print("| # runs | base (MB) | peak (MB) | diff (MB) |")
    print("| ------ | --------- | --------- | --------- |")

    run(1)
    run(2)
    run(3)
    run(4)
    run(5)
    run(6)
    run(7)
    run(8)
    run(9)
    run(14)
    run(16)
    run(8)
```


### Error message

```shell
| # runs | base (MB) | peak (MB) | diff (MB) |
| ------ | --------- | --------- | --------- |
|   1    |   45.79   |   46.59   |   0.80    |
|   2    |   91.57   |   92.37   |   0.80    |
|   3    |  137.36   |  138.16   |   0.80    |
|   4    |  183.15   |  183.94   |   0.80    |
|   5    |  228.93   |  229.73   |   0.80    |
|   6    |  274.72   |  275.52   |   0.80    |
|   7    |   45.79   |  321.21   |  275.42   |
|   8    |  366.29   |  367.08   |   0.80    |
|   9    |  412.07   |  412.87   |   0.80    |
|   14   |  641.00   |  641.80   |   0.80    |
|   16   |  366.29   |  412.07   |   45.78   |
|   8    |  366.29   |  367.08   |   0.80    |
```


### SciPy/NumPy/Python version and system information

```shell
python: 3.10.11
scipy: 1.11.1
```



Discussion:
Also tested with scipy==1.13.0
I'm able to reproduce this behavior on commit c106aa5b.

Observations:

* This doesn't seem to be solver-specific. SLSQP, BFGS, and L-BFGS-B all leak memory at essentially the same speed.

   This implies that whatever is going wrong is common to all solvers.
* The args array is about 2000 * 3000 * 8 bytes large, which is about [45.7 MiB](https://www.wolframalpha.com/input?i=2000*3000*8bytes+in+mebibytes), which about how much the base memory usage increases by in each row of that table. This suggests that minimize() is keeping the args argument referenced somehow.
* Similarly, increasing the size of args (e.g. `np.random.rand(2000, 30000)`) makes the leak faster.
* Examining referrers to args, it seems it's being kept alive by ScalarFunction. This is strange, since ScalarFunction shouldn't live past the end of the minimization function. In fact, we have many ScalarFunction objects.

Here is a test program I wrote which runs the previous reproducer, and counts the number of ScalarFunction objects on the heap. It goes up each time by the number of times that minimize() has been called, except (I think) when garbage collection is run.

```
import tracemalloc

from scipy.optimize import minimize
import numpy as np
import gc
from scipy.optimize._differentiable_functions import ScalarFunction


def do_something(*args, **kwargs):
    return np.random.rand()



def do_train():
    a = np.random.rand(2000, 3000)
    minimize(
        do_something,
        np.random.rand(300),
        args=(a,),
        method="L-BFGS-B"
    )
    obj = a

    #for i in range(4):
    #    ref_list = gc.get_referrers(obj)
    #    print(ref_list)
    #    if i != 2:
    #        obj = ref_list[0]
    #    else:
    #        obj = ref_list[0][2]
    #    del ref_list


def run(reps):
    tracemalloc.start()
    for _ in range(reps):
        do_train()

    base, peak = tracemalloc.get_traced_memory()
    tracemalloc.stop()
    sf = 0
    for obj in gc.get_objects():
        if isinstance(obj, ScalarFunction):
            sf += 1
    print(f"| {reps:^6} | {base/ 1024**2:^9.2f} | {peak/ 1024**2:^9.2f} | {(peak-base)/ 1024**2:^9.2f} | {sf:^7} |")


if __name__ == "__main__":
    print("| # runs | base (MB) | peak (MB) | diff (MB) | SF objs |")
    print("| ------ | --------- | --------- | --------- | ------- |")

    run(1)
    run(2)
    run(3)
    run(4)
    run(5)
    run(6)
    run(7)
    run(8)
    run(9)
    run(14)
    run(16)
    run(8)
```

Output:

```
| # runs | base (MB) | peak (MB) | diff (MB) | SF objs |
| ------ | --------- | --------- | --------- | ------- |
|   1    |   45.80   |   45.91   |   0.12    |    1    |
|   2    |   91.57   |   91.69   |   0.12    |    3    |
|   3    |  137.36   |  137.48   |   0.12    |    6    |
|   4    |  183.15   |  183.26   |   0.12    |   10    |
|   5    |  228.93   |  229.05   |   0.12    |   15    |
|   6    |  274.72   |  274.84   |   0.12    |   21    |
|   7    |  320.50   |  320.62   |   0.12    |   28    |
|   8    |  320.50   |  320.62   |   0.12    |    7    |
|   9    |  412.07   |  412.19   |   0.12    |   16    |
|   14   |  641.00   |  641.12   |   0.12    |   30    |
|   16   |  274.72   |  503.67   |  228.95   |    7    |
|   8    |  366.29   |  366.40   |   0.12    |   15    |
``` 
I think there's some kind of reference cycle inside `ScalarFunction`. I found it was possible to stop memory usage from growing by setting the function attributes that `ScalarFunction` defines inside `ScalarFunction.__init__()` to None just before the `sf` variable goes out of scope.

Here's the diff:

```
diff --git a/scipy/optimize/_lbfgsb_py.py b/scipy/optimize/_lbfgsb_py.py
index 42ad9038e..e15818db2 100644
--- a/scipy/optimize/_lbfgsb_py.py
+++ b/scipy/optimize/_lbfgsb_py.py
@@ -441,6 +441,10 @@ def _minimize_lbfgsb(fun, x0, args=(), jac=None, bounds=None,
     hess_inv = LbfgsInvHessProduct(s[:n_corrs], y[:n_corrs])
 
     task_str = task_str.decode()
+    sf._update_fun_impl = None
+    sf._update_grad_impl = None
+    sf._update_hess_impl = None
+    sf._update_x_impl = None
     return OptimizeResult(fun=f, jac=g, nfev=sf.nfev,
                           njev=sf.ngev,
                           nit=n_iterations, status=warnflag, message=task_str,
 ```

(This is not a serious suggestion for a fix - this fix would be ugly and need to be applied in every place which uses ScalarFunction. Better to fix it in one place.)

Result:

```
| # runs | base (MB) | peak (MB) | diff (MB) | SF objs |
| ------ | --------- | --------- | --------- | ------- |
|   1    |   0.01    |   45.91   |   45.90   |    0    |
|   2    |   0.00    |   45.90   |   45.90   |    0    |
|   3    |   0.00    |   45.90   |   45.90   |    0    |
|   4    |   0.00    |   45.90   |   45.90   |    0    |
|   5    |   0.00    |   45.90   |   45.90   |    0    |
|   6    |   0.00    |   45.90   |   45.90   |    0    |
|   7    |   0.00    |   45.90   |   45.90   |    0    |
|   8    |   0.00    |   45.90   |   45.90   |    0    |
|   9    |   0.00    |   45.90   |   45.90   |    0    |
|   14   |   0.00    |   45.90   |   45.90   |    0    |
|   16   |   0.00    |   45.90   |   45.90   |    0    |
|   8    |   0.00    |   45.90   |   45.90   |    0    |
```

My best guess is that the functions defined inside `ScalarFunction.__init__()` have a closure that allows them to access local variables, such as `self`. Then those functions get assigned as instance variables of ScalarFunction, which creates the reference cycle.
> thee functions defined inside ScalarFunction.__init__() have a closure that allows them to access local variables, such as self. Then those functions get assigned as instance variables of ScalarFunction

Those statements are correct, and I think that your deduction of the creation of the reference cycle is correct. It should be possible to amend `ScalarFunction` (and VectorFunction and LinearVectorFunction) to avoid this. However, I don't think it'll look as neat as it does now.

It's also worth stating that this has been around for several releases, so it's a wonder that it's not been picked up earlier.
> Those statements are correct, and I think that your deduction of the creation of the reference cycle is correct. It should be possible to amend `ScalarFunction` (and VectorFunction and LinearVectorFunction) to avoid this. However, I don't think it'll look as neat as it does now.

Yeah. I've been trying to think about how to solve this without compromising code quality. I have three ideas for how to fix the reference cycle.

1. Move each function from `__init__()`, and add it as an instance method. Rename each function to avoid duplicates. e.g. [this `update_grad`](https://github.com/scipy/scipy/blob/95e27a8573a7e22a9b6b70d8bbc372d778001f89/scipy/optimize/_differentiable_functions.py#L178) call might be renamed to a method called `_update_grad_impl_with_numdiff`. Then `__init__()` would select that implementation by running `self._update_grad_impl = self._update_grad_impl_with_numdiff`.

   Cons: moves the condition controlling if the code is run away from the code that it runs.
2. Move each function from `__init__()`, into module scope, and use something like [WeakMethod here](https://stackoverflow.com/a/55414240/530160) to turn them into instance methods.

   Cons: don't know performance impact of weakref
3. The code above breaks the reference cycle by deleting all of the functions which have a reference to `self`. We could create a method called `ScalarFunction.close()`, which clears those four functions, which would allow the garbage collector to collect ScalarFunction when it goes out of scope.

   Then, we would go to every place where ScalarFunction is used, and add a call to `ScalarFunction.close()`.

   Cons: This would require 30 calls to ScalarFunction.close(). Every future use of this in the codebase would need it too, or it could create a new leak.

I'm not sure what approach would be best. Thoughts?
Option 1 seems the most natural to me. The current way (setting object attribute equal to local function with closure capture) looks like an anti-pattern to me.

An “object-oriented” approach may be to subclass this class depending on e.g. whether the Hessian is supplied. The user code would call a factory method instead of constructing a `ScalarFunction` directly. That would be more “proper” in my opinion, but it requires larger refactoring.

Resolving patch:
diff --git a/scipy/optimize/_differentiable_functions.py b/scipy/optimize/_differentiable_functions.py
--- a/scipy/optimize/_differentiable_functions.py
+++ b/scipy/optimize/_differentiable_functions.py
@@ -9,6 +9,84 @@
 FD_METHODS = ('2-point', '3-point', 'cs')
 
 
+def _wrapper_fun(fun, args=()):
+    ncalls = [0]
+
+    def wrapped(x):
+        ncalls[0] += 1
+        # Send a copy because the user may overwrite it.
+        # Overwriting results in undefined behaviour because
+        # fun(self.x) will change self.x, with the two no longer linked.
+        fx = fun(np.copy(x), *args)
+        # Make sure the function returns a true scalar
+        if not np.isscalar(fx):
+            try:
+                fx = np.asarray(fx).item()
+            except (TypeError, ValueError) as e:
+                raise ValueError(
+                    "The user-provided objective function "
+                    "must return a scalar value."
+                ) from e
+        return fx
+    return wrapped, ncalls
+
+
+def _wrapper_grad(grad, fun=None, args=(), finite_diff_options=None):
+    ncalls = [0]
+
+    if callable(grad):
+        def wrapped(x, **kwds):
+            # kwds present to give function same signature as numdiff variant
+            ncalls[0] += 1
+            return np.atleast_1d(grad(np.copy(x), *args))
+        return wrapped, ncalls
+
+    elif grad in FD_METHODS:
+        def wrapped1(x, f0=None):
+            ncalls[0] += 1
+            return approx_derivative(
+                fun, x, f0=f0, **finite_diff_options
+            )
+
+        return wrapped1, ncalls
+
+
+def _wrapper_hess(hess, grad=None, x0=None, args=(), finite_diff_options=None):
+    if callable(hess):
+        H = hess(np.copy(x0), *args)
+        ncalls = [1]
+
+        if sps.issparse(H):
+            def wrapped(x, **kwds):
+                ncalls[0] += 1
+                return sps.csr_matrix(hess(np.copy(x), *args))
+
+            H = sps.csr_matrix(H)
+
+        elif isinstance(H, LinearOperator):
+            def wrapped(x, **kwds):
+                ncalls[0] += 1
+                return hess(np.copy(x), *args)
+
+        else:  # dense
+            def wrapped(x, **kwds):
+                ncalls[0] += 1
+                return np.atleast_2d(np.asarray(hess(np.copy(x), *args)))
+
+            H = np.atleast_2d(np.asarray(H))
+
+        return wrapped, ncalls, H
+    elif hess in FD_METHODS:
+        ncalls = [0]
+
+        def wrapped1(x, f0=None):
+            return approx_derivative(
+                grad, x, f0=f0, **finite_diff_options
+            )
+
+        return wrapped1, ncalls, None
+
+
 class ScalarFunction:
     """Scalar function and its derivatives.
 
@@ -110,13 +188,17 @@ def __init__(self, fun, x0, args, grad, hess, finite_diff_rel_step,
         if xp.isdtype(_x.dtype, "real floating"):
             _dtype = _x.dtype
 
+        # original arguments
+        self._wrapped_fun, self._nfev = _wrapper_fun(fun, args=args)
+        self._orig_fun = fun
+        self._orig_grad = grad
+        self._orig_hess = hess
+        self._args = args
+
         # promotes to floating
         self.x = xp.astype(_x, _dtype)
         self.x_dtype = _dtype
         self.n = self.x.size
-        self.nfev = 0
-        self.ngev = 0
-        self.nhev = 0
         self.f_updated = False
         self.g_updated = False
         self.H_updated = False
@@ -136,88 +218,33 @@ def __init__(self, fun, x0, args, grad, hess, finite_diff_rel_step,
             finite_diff_options["abs_step"] = epsilon
             finite_diff_options["as_linear_operator"] = True
 
-        # Function evaluation
-        def fun_wrapped(x):
-            self.nfev += 1
-            # Send a copy because the user may overwrite it.
-            # Overwriting results in undefined behaviour because
-            # fun(self.x) will change self.x, with the two no longer linked.
-            fx = fun(np.copy(x), *args)
-            # Make sure the function returns a true scalar
-            if not np.isscalar(fx):
-                try:
-                    fx = np.asarray(fx).item()
-                except (TypeError, ValueError) as e:
-                    raise ValueError(
-                        "The user-provided objective function "
-                        "must return a scalar value."
-                    ) from e
-
-            if fx < self._lowest_f:
-                self._lowest_x = x
-                self._lowest_f = fx
-
-            return fx
-
-        def update_fun():
-            self.f = fun_wrapped(self.x)
-
-        self._update_fun_impl = update_fun
+        # Initial function evaluation
         self._update_fun()
 
-        # Gradient evaluation
-        if callable(grad):
-            def grad_wrapped(x):
-                self.ngev += 1
-                return np.atleast_1d(grad(np.copy(x), *args))
-
-            def update_grad():
-                self.g = grad_wrapped(self.x)
-
-        elif grad in FD_METHODS:
-            def update_grad():
-                self._update_fun()
-                self.ngev += 1
-                self.g = approx_derivative(fun_wrapped, self.x, f0=self.f,
-                                           **finite_diff_options)
-
-        self._update_grad_impl = update_grad
+        # Initial gradient evaluation
+        self._wrapped_grad, self._ngev = _wrapper_grad(
+            grad,
+            fun=self._wrapped_fun,
+            args=args,
+            finite_diff_options=finite_diff_options
+        )
         self._update_grad()
 
-        # Hessian Evaluation
+        # Hessian evaluation
         if callable(hess):
-            self.H = hess(np.copy(x0), *args)
+            self._wrapped_hess, self._nhev, self.H = _wrapper_hess(
+                hess, x0=x0, args=args
+            )
             self.H_updated = True
-            self.nhev += 1
-
-            if sps.issparse(self.H):
-                def hess_wrapped(x):
-                    self.nhev += 1
-                    return sps.csr_matrix(hess(np.copy(x), *args))
-                self.H = sps.csr_matrix(self.H)
-
-            elif isinstance(self.H, LinearOperator):
-                def hess_wrapped(x):
-                    self.nhev += 1
-                    return hess(np.copy(x), *args)
-
-            else:
-                def hess_wrapped(x):
-                    self.nhev += 1
-                    return np.atleast_2d(np.asarray(hess(np.copy(x), *args)))
-                self.H = np.atleast_2d(np.asarray(self.H))
-
-            def update_hess():
-                self.H = hess_wrapped(self.x)
-
         elif hess in FD_METHODS:
-            def update_hess():
-                self._update_grad()
-                self.H = approx_derivative(grad_wrapped, self.x, f0=self.g,
-                                           **finite_diff_options)
-                return self.H
-
-            update_hess()
+            self._wrapped_hess, self._nhev, self.H = _wrapper_hess(
+                hess,
+                grad=self._wrapped_grad,
+                x0=x0,
+                finite_diff_options=finite_diff_options
+            )
+            self._update_grad()
+            self.H = self._wrapped_hess(self.x, f0=self.g)
             self.H_updated = True
         elif isinstance(hess, HessianUpdateStrategy):
             self.H = hess
@@ -225,74 +252,94 @@ def update_hess():
             self.H_updated = True
             self.x_prev = None
             self.g_prev = None
+            self._nhev = [0]
 
-            def update_hess():
-                self._update_grad()
-                self.H.update(self.x - self.x_prev, self.g - self.g_prev)
+    @property
+    def nfev(self):
+        return self._nfev[0]
 
-        self._update_hess_impl = update_hess
+    @property
+    def ngev(self):
+        return self._ngev[0]
 
-        if isinstance(hess, HessianUpdateStrategy):
-            def update_x(x):
-                self._update_grad()
-                self.x_prev = self.x
-                self.g_prev = self.g
-                # ensure that self.x is a copy of x. Don't store a reference
-                # otherwise the memoization doesn't work properly.
+    @property
+    def nhev(self):
+        return self._nhev[0]
 
-                _x = atleast_nd(x, ndim=1, xp=self.xp)
-                self.x = self.xp.astype(_x, self.x_dtype)
-                self.f_updated = False
-                self.g_updated = False
-                self.H_updated = False
-                self._update_hess()
+    def _update_x(self, x):
+        if isinstance(self._orig_hess, HessianUpdateStrategy):
+            self._update_grad()
+            self.x_prev = self.x
+            self.g_prev = self.g
+            # ensure that self.x is a copy of x. Don't store a reference
+            # otherwise the memoization doesn't work properly.
+
+            _x = atleast_nd(x, ndim=1, xp=self.xp)
+            self.x = self.xp.astype(_x, self.x_dtype)
+            self.f_updated = False
+            self.g_updated = False
+            self.H_updated = False
+            self._update_hess()
         else:
-            def update_x(x):
-                # ensure that self.x is a copy of x. Don't store a reference
-                # otherwise the memoization doesn't work properly.
-                _x = atleast_nd(x, ndim=1, xp=self.xp)
-                self.x = self.xp.astype(_x, self.x_dtype)
-                self.f_updated = False
-                self.g_updated = False
-                self.H_updated = False
-        self._update_x_impl = update_x
+            # ensure that self.x is a copy of x. Don't store a reference
+            # otherwise the memoization doesn't work properly.
+            _x = atleast_nd(x, ndim=1, xp=self.xp)
+            self.x = self.xp.astype(_x, self.x_dtype)
+            self.f_updated = False
+            self.g_updated = False
+            self.H_updated = False
 
     def _update_fun(self):
         if not self.f_updated:
-            self._update_fun_impl()
+            fx = self._wrapped_fun(self.x)
+            if fx < self._lowest_f:
+                self._lowest_x = self.x
+                self._lowest_f = fx
+
+            self.f = fx
             self.f_updated = True
 
     def _update_grad(self):
         if not self.g_updated:
-            self._update_grad_impl()
+            if self._orig_grad in FD_METHODS:
+                self._update_fun()
+            self.g = self._wrapped_grad(self.x, f0=self.f)
             self.g_updated = True
 
     def _update_hess(self):
         if not self.H_updated:
-            self._update_hess_impl()
+            if self._orig_hess in FD_METHODS:
+                self._update_grad()
+                self.H = self._wrapped_hess(self.x, f0=self.g)
+            elif isinstance(self._orig_hess, HessianUpdateStrategy):
+                self._update_grad()
+                self.H.update(self.x - self.x_prev, self.g - self.g_prev)
+            else:       # should be callable(hess)
+                self.H = self._wrapped_hess(self.x)
+
             self.H_updated = True
 
     def fun(self, x):
         if not np.array_equal(x, self.x):
-            self._update_x_impl(x)
+            self._update_x(x)
         self._update_fun()
         return self.f
 
     def grad(self, x):
         if not np.array_equal(x, self.x):
-            self._update_x_impl(x)
+            self._update_x(x)
         self._update_grad()
         return self.g
 
     def hess(self, x):
         if not np.array_equal(x, self.x):
-            self._update_x_impl(x)
+            self._update_x(x)
         self._update_hess()
         return self.H
 
     def fun_and_grad(self, x):
         if not np.array_equal(x, self.x):
-            self._update_x_impl(x)
+            self._update_x(x)
         self._update_fun()
         self._update_grad()
         return self.f, self.g


Test patch:
diff --git a/scipy/optimize/tests/test_differentiable_functions.py b/scipy/optimize/tests/test_differentiable_functions.py
--- a/scipy/optimize/tests/test_differentiable_functions.py
+++ b/scipy/optimize/tests/test_differentiable_functions.py
@@ -1,8 +1,10 @@
 import pytest
+import platform
 import numpy as np
 from numpy.testing import (TestCase, assert_array_almost_equal,
                            assert_array_equal, assert_, assert_allclose,
                            assert_equal)
+from scipy._lib._gcutils import assert_deallocated
 from scipy.sparse import csr_matrix
 from scipy.sparse.linalg import LinearOperator
 from scipy.optimize._differentiable_functions import (ScalarFunction,
@@ -756,3 +758,46 @@ def test_IdentityVectorFunction():
     assert_array_equal(f2.jac(x), np.eye(3))
 
     assert_array_equal(f1.hess(x, v).toarray(), np.zeros((3, 3)))
+
+
+@pytest.mark.skipif(
+    platform.python_implementation() == "PyPy",
+    reason="assert_deallocate not available on PyPy"
+)
+def test_ScalarFunctionNoReferenceCycle():
+    """Regression test for gh-20768."""
+    ex = ExScalarFunction()
+    x0 = np.zeros(3)
+    with assert_deallocated(lambda: ScalarFunction(ex.fun, x0, (), ex.grad,
+                            ex.hess, None, (-np.inf, np.inf))):
+        pass
+
+
+@pytest.mark.skipif(
+    platform.python_implementation() == "PyPy",
+    reason="assert_deallocate not available on PyPy"
+)
+@pytest.mark.xfail(reason="TODO remove reference cycle from VectorFunction")
+def test_VectorFunctionNoReferenceCycle():
+    """Regression test for gh-20768."""
+    ex = ExVectorialFunction()
+    x0 = [1.0, 0.0]
+    with assert_deallocated(lambda: VectorFunction(ex.fun, x0, ex.jac,
+                            ex.hess, None, None, (-np.inf, np.inf), None)):
+        pass
+
+
+@pytest.mark.skipif(
+    platform.python_implementation() == "PyPy",
+    reason="assert_deallocate not available on PyPy"
+)
+def test_LinearVectorFunctionNoReferenceCycle():
+    """Regression test for gh-20768."""
+    A_dense = np.array([
+        [-1, 2, 0],
+        [0, 4, 2]
+    ])
+    x0 = np.zeros(3)
+    A_sparse = csr_matrix(A_dense)
+    with assert_deallocated(lambda: LinearVectorFunction(A_sparse, x0, None)):
+        pass