Repository: scipy/scipy

Issue:
BUG: scipy.optimize.minimize: Powell's method function evaluated outside of bound
### Describe your issue.

When I run scipy.optimize.minimize with Powell option and defined bounds for a two-variable function, it evaluates the function even outside of the bounds.

### Reproducing Code Example

```python
count = 1
def fun(x):
    global count
    a, b = x
    print(count)
    print(x)
    count += 1
    return a**2 + b**2
scipy.optimize.minimize(fun, x0=[0.6,20], method='Powell', bounds=[(0,0.6),(0,20)], options = {'xtol':0.001, 'ftol':0.001, 'return_all':True, 'disp':True})
```


### Error message

```shell
It goes outside of bound at 39th function evaluation
```


### SciPy/NumPy/Python version information

1.7.3 1.20.1 sys.version_info(major=3, minor=8, micro=8, releaselevel='final', serial=0)


Discussion:
@jtiosue do you have a comment on how to fix this issue? It's also present in the main branch.
It looks like the extrapolated point lies outside the bounds:

```
import numpy as np
from scipy.optimize import minimize, Bounds
bounds = Bounds([0, 0], [0.6, 20])
counts = [0]
def fun(x):
    a, b = x
    print(counts[0], x)
    counts[0] += 1

    assert (x >= bounds.lb).all() and (x <= bounds.ub).all()
    return a**2 + b**2


minimize(fun, x0=[0.6,20], method='Powell', bounds=[(0, 0.6),(0,20)], options = {'xtol':0.001, 'ftol':0.001, 'return_all':True, 'disp':True})
```
```
---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
/var/folders/m8/gwtcncws12jf60xw5n6knwnw0000gn/T/ipykernel_14908/899199065.py in <module>
     10 
     11 
---> 12 minimize(fun, x0=[0.6,20], method='Powell', bounds=[(0, 0.6),(0,20)], options = {'xtol':0.001, 'ftol':0.001, 'return_all':True, 'disp':True})

~/Documents/Andy/programming/scipy/installdir/lib/python3.9/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
    670                                    **options)
    671     elif meth == 'powell':
--> 672         res = _minimize_powell(fun, x0, args, callback, bounds, **options)
    673     elif meth == 'cg':
    674         res = _minimize_cg(fun, x0, args, jac, callback, **options)

~/Documents/Andy/programming/scipy/installdir/lib/python3.9/site-packages/scipy/optimize/_optimize.py in _minimize_powell(func, x0, args, callback, bounds, xtol, ftol, maxiter, maxfev, disp, direc, return_all, **unknown_options)
   3221             x2 = 2*x - x1
   3222             x1 = x.copy()
-> 3223             fx2 = squeeze(func(x2))
   3224 
   3225             if (fx > fx2):

~/Documents/Andy/programming/scipy/installdir/lib/python3.9/site-packages/scipy/optimize/_optimize.py in function_wrapper(x, *wrapper_args)
    494         ncalls[0] += 1
    495         # A copy of x is sent to the user function (gh13740)
--> 496         fx = function(np.copy(x), *(wrapper_args + args))
    497         # Ideally, we'd like to a have a true scalar returned from f(x). For
    498         # backwards-compatibility, also allow np.array([1.3]),

/var/folders/m8/gwtcncws12jf60xw5n6knwnw0000gn/T/ipykernel_14908/899199065.py in fun(x)
      6     counts[0] += 1
      7 
----> 8     assert (x >= bounds.lb).all() and (x <= bounds.ub).all()
      9     return a**2 + b**2
     10 

AssertionError: 
```
@andyfaff I'm sorry I don't have time to submit a full pull request right now or make new tests. But I believe the issue is with lines 3219 through 3223 in the file `scipy.optimize._optimize.py` (inside the function `_minimize_powell`). It currently is:

```
# Construct the extrapolated point
direc1 = x - x1
x2 = 2*x - x1
x1 = x.copy()
fx2 = squeeze(func(x2))
```

What it should instead say is:

```
# Construct the extrapolated point
direc1 = x - x1
x1 = x.copy()
_, lmax = _line_for_search(x, direc1, lower_bound, upper_bound)
x2 = x + min(lmax, 1) * direc1
fx2 = squeeze(func(x2))
```

Thank you for looking into this matter.
Is there a way for me to fix this bug on my end?
Or is it something that you have to do on your end and update the whole library?
Thanks

Resolving patch:
diff --git a/scipy/optimize/_optimize.py b/scipy/optimize/_optimize.py
--- a/scipy/optimize/_optimize.py
+++ b/scipy/optimize/_optimize.py
@@ -3259,13 +3259,12 @@ def _minimize_powell(func, x0, args=(), callback=None, bounds=None,
     fval = squeeze(func(x))
     x1 = x.copy()
     iter = 0
-    ilist = list(range(N))
     while True:
         try:
             fx = fval
             bigind = 0
             delta = 0.0
-            for i in ilist:
+            for i in range(N):
                 direc1 = direc[i]
                 fx2 = fval
                 fval, x, direc1 = _linesearch_powell(func, x, direc1,
@@ -3294,8 +3293,13 @@ def _minimize_powell(func, x0, args=(), callback=None, bounds=None,
 
             # Construct the extrapolated point
             direc1 = x - x1
-            x2 = 2*x - x1
             x1 = x.copy()
+            # make sure that we don't go outside the bounds when extrapolating
+            if lower_bound is None and upper_bound is None:
+                lmax = 1
+            else:
+                _, lmax = _line_for_search(x, direc1, lower_bound, upper_bound)
+            x2 = x + min(lmax, 1) * direc1
             fx2 = squeeze(func(x2))
 
             if (fx > fx2):


Test patch:
diff --git a/scipy/optimize/tests/test_optimize.py b/scipy/optimize/tests/test_optimize.py
--- a/scipy/optimize/tests/test_optimize.py
+++ b/scipy/optimize/tests/test_optimize.py
@@ -308,17 +308,15 @@ def test_powell_bounded(self):
 
             assert func_calls == self.funccalls
             assert_allclose(self.func(params), self.func(self.solution),
-                            atol=1e-6)
+                            atol=1e-6, rtol=1e-5)
 
-            # Ensure that function call counts are 'known good'.
-            # Generally, this takes 131 function calls. However, on some CI
-            # checks it finds 138 funccalls. This 20 call leeway was also
-            # included in the test_powell function.
             # The exact evaluation count is sensitive to numerical error, and
             # floating-point computations are not bit-for-bit reproducible
             # across machines, and when using e.g. MKL, data alignment etc.
             # affect the rounding error.
-            assert self.funccalls <= 131 + 20
+            # It takes 155 calls on my machine, but we can add the same +20
+            # margin as is used in `test_powell`
+            assert self.funccalls <= 155 + 20
             assert self.gradcalls == 0
 
     def test_neldermead(self):
@@ -2019,6 +2017,18 @@ def test_linesearch_powell_bounded():
         assert_allclose(direction, l * xi, atol=1e-6)
 
 
+def test_powell_limits():
+    # gh15342 - powell was going outside bounds for some function evaluations.
+    bounds = optimize.Bounds([0, 0], [0.6, 20])
+
+    def fun(x):
+        a, b = x
+        assert (x >= bounds.lb).all() and (x <= bounds.ub).all()
+        return a ** 2 + b ** 2
+
+    optimize.minimize(fun, x0=[0.6, 20], method='Powell', bounds=bounds)
+
+
 class TestRosen:
 
     def test_hess(self):