Repository: scipy/scipy

Issue:
Function always evaluated twice when using HessianUpdateStrategy with function that returns value and gradient
<!-- 
Thank you for taking the time to file a bug report. 
Please fill in the fields below, deleting the sections that 
don't apply to your issue. You can view the final output
by clicking the preview button above.
Note: This is a comment, and won't appear in the output.
-->

[scipy.optimize.minimize](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html) supports handing a function that computes both function value and jacobian and returns them as tuple. I noticed that when using `method='trusts-constr'`, this results in the function always being called twice in a row for exactly the same value. After digging into the code, I noticed that the function is wrapped into `MemoizeJac` which should avoid calling the function multiple times, but this doesn't work in this case:

#### Reproducing code example:
```
import numpy as np
from scipy.optimize._differentiable_functions import ScalarFunction
from scipy.optimize._minimize import MemoizeJac
from scipy.optimize import BFGS

def test_fun(x):
    print("eval", x)
    return np.sum(x**2), 2*x

fun = MemoizeJac(test_fun)
jac = fun.derivative
hess = BFGS()

x0 = np.array([1.0, 0.5])
# the following line will evaluate the function once
diff_fun = ScalarFunction(fun, x0, (), jac, hess, None, None) 

# the following line will evaluate the function _twice_
diff_fun.fun(x0+0.1)

# as will this line
diff_fun.fun(x0+0.2) 
```
#### Error

`test_fun` should have been called only once per function evaluation, but except for the initialization of ScalarFunction, it is always called twice. I think I already found the source of this: `MemoizeJac` assumes that the function value is always requested before the gradient value and therefore only checks in the gradient call for existing results. However, if the hessian is a `UpdateHessingStrategy`, calling `diff.fun` first requests the gradient in order to do a hessian update and only then requests the function value. And in this case, `MemoizeJac` does not cache the function result.

#### Scipy/Numpy/Python version information:
```
numpy: 1.3.3 scipy: 1.17.4 system: sys.version_info(major=3, minor=6, micro=8, releaselevel='final', serial=0)
```


Discussion:

Resolving patch:
diff --git a/scipy/optimize/optimize.py b/scipy/optimize/optimize.py
--- a/scipy/optimize/optimize.py
+++ b/scipy/optimize/optimize.py
@@ -51,26 +51,30 @@
 
 
 class MemoizeJac(object):
-    """ Decorator that caches the value gradient of function each time it
-    is called. """
+    """ Decorator that caches the return values of a function returning `(fun, grad)`
+        each time it is called. """
 
     def __init__(self, fun):
         self.fun = fun
         self.jac = None
+        self._value = None
         self.x = None
 
+    def _compute_if_needed(self, x, *args):
+        if not np.all(x == self.x) or self._value is None or self.jac is None:
+            self.x = np.asarray(x).copy()
+            fg = self.fun(x, *args)
+            self.jac = fg[1]
+            self._value = fg[0]
+
     def __call__(self, x, *args):
-        self.x = np.asarray(x).copy()
-        fg = self.fun(x, *args)
-        self.jac = fg[1]
-        return fg[0]
+        """ returns the the function value """
+        self._compute_if_needed(x, *args)
+        return self._value
 
     def derivative(self, x, *args):
-        if self.jac is not None and np.all(x == self.x):
-            return self.jac
-        else:
-            self(x, *args)
-            return self.jac
+        self._compute_if_needed(x, *args)
+        return self.jac
 
 
 class OptimizeResult(dict):


Test patch:
diff --git a/scipy/optimize/tests/test_optimize.py b/scipy/optimize/tests/test_optimize.py
--- a/scipy/optimize/tests/test_optimize.py
+++ b/scipy/optimize/tests/test_optimize.py
@@ -22,6 +22,7 @@
 from scipy import optimize
 from scipy.optimize._minimize import MINIMIZE_METHODS
 from scipy.optimize._differentiable_functions import ScalarFunction
+from scipy.optimize.optimize import MemoizeJac
 
 
 def test_check_grad():
@@ -1666,3 +1667,76 @@ def hess(x):
         res = optimize.minimize(fun, np.array([0.1]), method=method, jac=jac,
                                 hess=hess)
         assert res.x.shape == (1,)
+
+
+class FunctionWithGradient(object):
+    def __init__(self):
+        self.number_of_calls = 0
+
+    def __call__(self, x):
+        self.number_of_calls += 1
+        return np.sum(x**2), 2 * x
+
+
+@pytest.fixture
+def function_with_gradient():
+    return FunctionWithGradient()
+
+
+def test_memoize_jac_function_before_gradient(function_with_gradient):
+    memoized_function = MemoizeJac(function_with_gradient)
+
+    x0 = np.array([1.0, 2.0])
+    assert_allclose(memoized_function(x0), 5.0)
+    assert function_with_gradient.number_of_calls == 1
+
+    assert_allclose(memoized_function.derivative(x0), 2 * x0)
+    assert function_with_gradient.number_of_calls == 1, \
+        "function is not recomputed " \
+        "if gradient is requested after function value"
+
+    assert_allclose(
+        memoized_function(2 * x0), 20.0,
+        err_msg="different input triggers new computation")
+    assert function_with_gradient.number_of_calls == 2, \
+        "different input triggers new computation"
+
+
+def test_memoize_jac_gradient_before_function(function_with_gradient):
+    memoized_function = MemoizeJac(function_with_gradient)
+
+    x0 = np.array([1.0, 2.0])
+    assert_allclose(memoized_function.derivative(x0), 2 * x0)
+    assert function_with_gradient.number_of_calls == 1
+
+    assert_allclose(memoized_function(x0), 5.0)
+    assert function_with_gradient.number_of_calls == 1, \
+        "function is not recomputed " \
+        "if function value is requested after gradient"
+
+    assert_allclose(
+        memoized_function.derivative(2 * x0), 4 * x0,
+        err_msg="different input triggers new computation")
+    assert function_with_gradient.number_of_calls == 2, \
+        "different input triggers new computation"
+
+
+def test_memoize_jac_with_bfgs(function_with_gradient):
+    """ Tests that using MemoizedJac in combination with ScalarFunction
+        and BFGS does not lead to repeated function evaluations.
+        Tests changes made in response to GH11868.
+    """
+    memoized_function = MemoizeJac(function_with_gradient)
+    jac = memoized_function.derivative
+    hess = optimize.BFGS()
+
+    x0 = np.array([1.0, 0.5])
+    scalar_function = ScalarFunction(
+        memoized_function, x0, (), jac, hess, None, None)
+    assert function_with_gradient.number_of_calls == 1
+
+    scalar_function.fun(x0 + 0.1)
+    assert function_with_gradient.number_of_calls == 2
+
+    scalar_function.fun(x0 + 0.2)
+    assert function_with_gradient.number_of_calls == 3