Repository: scipy/scipy

Issue:
Feature request: Nelder-Mead with bounds
Currently `scipy.optimize.minimize` provides no gradient-free optimization schemes for bounded problems. Nelder-Mead is perhaps the most frequently used such algorithm, and it can work with bounds too:
At expansion or reflection steps, the variables need to be projected to be within bounds. For description see Appendix A of "A constrained, globalized, and bounded Nelder–Mead method
for engineering optimization".


Discussion:
While not `minimize` differential_evolutuin will do what you require.
> While not `minimize` differential_evolutuin will do what you require.

True, but that's a global method... if only local minimization is required, this should converge much faster.

https://www.google.com/url?sa=t&source=web&rct=j&url=https://www.emse.fr/~leriche/GBNM_SMO_1026_final.pdf&ved=2ahUKEwizuuntypPnAhW04HMBHd43B4YQFjABegQIBRAB&usg=AOvVaw0Ox-n6PW-v7Uq2OMUyXlYo&cshid=1579571904651
Sorry for the poor formatting. It's a link to the paper. As far as I can tell it's a simple clip to bounds.
What is the status on implementing the Globalized Bounded Nelder–Mead algorithm from the paper link above? I am at a deadend and hoping that this will be available soon.
As far as I know no one is currently working on this. Pull requests are welcomed.

Resolving patch:
diff --git a/scipy/optimize/_minimize.py b/scipy/optimize/_minimize.py
--- a/scipy/optimize/_minimize.py
+++ b/scipy/optimize/_minimize.py
@@ -144,7 +144,7 @@ def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
         dimension (n,) and `args` is a tuple with the fixed
         parameters.
     bounds : sequence or `Bounds`, optional
-        Bounds on variables for L-BFGS-B, TNC, SLSQP, Powell, and
+        Bounds on variables for Nelder-Mead, L-BFGS-B, TNC, SLSQP, Powell, and
         trust-constr methods. There are two ways to specify the bounds:
 
             1. Instance of `Bounds` class.
@@ -532,11 +532,11 @@ def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
         warn('Method %s does not use Hessian-vector product '
              'information (hessp).' % method, RuntimeWarning)
     # - constraints or bounds
-    if (meth in ('nelder-mead', 'cg', 'bfgs', 'newton-cg', 'dogleg',
-                 'trust-ncg') and (bounds is not None or np.any(constraints))):
+    if (meth in ('cg', 'bfgs', 'newton-cg', 'dogleg', 'trust-ncg')
+            and (bounds is not None or np.any(constraints))):
         warn('Method %s cannot handle constraints nor bounds.' % method,
              RuntimeWarning)
-    if meth in ('l-bfgs-b', 'tnc', 'powell') and np.any(constraints):
+    if meth in ('nelder-mead', 'l-bfgs-b', 'tnc', 'powell') and np.any(constraints):
         warn('Method %s cannot handle constraints.' % method,
              RuntimeWarning)
     if meth == 'cobyla' and bounds is not None:
@@ -607,7 +607,8 @@ def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
         constraints = standardize_constraints(constraints, x0, meth)
 
     if meth == 'nelder-mead':
-        return _minimize_neldermead(fun, x0, args, callback, **options)
+        return _minimize_neldermead(fun, x0, args, callback, bounds=bounds,
+                                    **options)
     elif meth == 'powell':
         return _minimize_powell(fun, x0, args, callback, bounds, **options)
     elif meth == 'cg':
@@ -808,7 +809,7 @@ def minimize_scalar(fun, bracket=None, bounds=None, args=(),
 
 def standardize_bounds(bounds, x0, meth):
     """Converts bounds to the form required by the solver."""
-    if meth in {'trust-constr', 'powell'}:
+    if meth in {'trust-constr', 'powell', 'nelder-mead'}:
         if not isinstance(bounds, Bounds):
             lb, ub = old_bound_to_new(bounds)
             bounds = Bounds(lb, ub)
diff --git a/scipy/optimize/optimize.py b/scipy/optimize/optimize.py
--- a/scipy/optimize/optimize.py
+++ b/scipy/optimize/optimize.py
@@ -593,7 +593,7 @@ def fmin(func, x0, args=(), xtol=1e-4, ftol=1e-4, maxiter=None, maxfun=None,
 def _minimize_neldermead(func, x0, args=(), callback=None,
                          maxiter=None, maxfev=None, disp=False,
                          return_all=False, initial_simplex=None,
-                         xatol=1e-4, fatol=1e-4, adaptive=False,
+                         xatol=1e-4, fatol=1e-4, adaptive=False, bounds=None,
                          **unknown_options):
     """
     Minimization of scalar function of one or more variables using the
@@ -626,6 +626,15 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
     adaptive : bool, optional
         Adapt algorithm parameters to dimensionality of problem. Useful for
         high-dimensional minimization [1]_.
+    bounds : sequence or `Bounds`, optional
+        Bounds on variables. There are two ways to specify the bounds:
+
+            1. Instance of `Bounds` class.
+            2. Sequence of ``(min, max)`` pairs for each element in `x`. None
+               is used to specify no bound.
+
+        Note that this just clips all vertices in simplex based on
+        the bounds.
 
     References
     ----------
@@ -679,6 +688,18 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
 
     x0 = asfarray(x0).flatten()
 
+    if bounds is not None:
+        lower_bound, upper_bound = bounds.lb, bounds.ub
+        # check bounds
+        if (lower_bound > upper_bound).any():
+            raise ValueError("Nelder Mead - one of the lower bounds is greater than an upper bound.")
+        if np.any(lower_bound > x0) or np.any(x0 > upper_bound):
+            warnings.warn("Initial guess is not within the specified bounds",
+                          OptimizeWarning, 3)
+
+    if bounds is not None:
+        x0 = np.clip(x0, lower_bound, upper_bound)
+
     if initial_simplex is None:
         N = len(x0)
 
@@ -719,6 +740,9 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
         else:
             maxfun = np.inf
 
+    if bounds is not None:
+        sim = np.clip(sim, lower_bound, upper_bound)
+
     one2np1 = list(range(1, N + 1))
     fsim = np.empty((N + 1,), float)
 
@@ -739,11 +763,15 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
 
         xbar = np.add.reduce(sim[:-1], 0) / N
         xr = (1 + rho) * xbar - rho * sim[-1]
+        if bounds is not None:
+            xr = np.clip(xr, lower_bound, upper_bound)
         fxr = func(xr)
         doshrink = 0
 
         if fxr < fsim[0]:
             xe = (1 + rho * chi) * xbar - rho * chi * sim[-1]
+            if bounds is not None:
+                xe = np.clip(xe, lower_bound, upper_bound)
             fxe = func(xe)
 
             if fxe < fxr:
@@ -760,6 +788,8 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
                 # Perform contraction
                 if fxr < fsim[-1]:
                     xc = (1 + psi * rho) * xbar - psi * rho * sim[-1]
+                    if bounds is not None:
+                        xc = np.clip(xc, lower_bound, upper_bound)
                     fxc = func(xc)
 
                     if fxc <= fxr:
@@ -770,6 +800,8 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
                 else:
                     # Perform an inside contraction
                     xcc = (1 - psi) * xbar + psi * sim[-1]
+                    if bounds is not None:
+                        xcc = np.clip(xcc, lower_bound, upper_bound)
                     fxcc = func(xcc)
 
                     if fxcc < fsim[-1]:
@@ -781,6 +813,8 @@ def _minimize_neldermead(func, x0, args=(), callback=None,
                 if doshrink:
                     for j in one2np1:
                         sim[j] = sim[0] + sigma * (sim[j] - sim[0])
+                        if bounds is not None:
+                            sim[j] = np.clip(sim[j], lower_bound, upper_bound)
                         fsim[j] = func(sim[j])
 
         ind = np.argsort(fsim)


Test patch:
diff --git a/scipy/optimize/tests/test_minimize_constrained.py b/scipy/optimize/tests/test_minimize_constrained.py
--- a/scipy/optimize/tests/test_minimize_constrained.py
+++ b/scipy/optimize/tests/test_minimize_constrained.py
@@ -4,8 +4,8 @@
 from scipy.sparse import csc_matrix
 from numpy.testing import (TestCase, assert_array_almost_equal,
                            assert_array_less, assert_,
-                           suppress_warnings)
-from pytest import raises
+                           suppress_warnings, assert_allclose)
+from pytest import raises, warns
 from scipy.optimize import (NonlinearConstraint,
                             LinearConstraint,
                             Bounds,
@@ -676,3 +676,65 @@ def opt(x):
         A = np.matrix(np.diag([1, 1]))
     lin_cons = LinearConstraint(A, -1, np.inf)
     minimize(opt, 2*[1], constraints = lin_cons)  # just checking that there are no errors
+
+
+class TestBoundedNelderMead:
+
+    @pytest.mark.parametrize('bounds, x_opt',
+                             [(Bounds(-np.inf, np.inf), Rosenbrock().x_opt),
+                              (Bounds(-np.inf, -0.8), [-0.8, -0.8]),
+                              (Bounds(3.0, np.inf), [3.0, 9.0]),
+                              (Bounds([3.0, 1.0], [4.0, 5.0]), [3., 5.]),
+                              ])
+    def test_rosen_brock_with_bounds(self, bounds, x_opt):
+        prob = Rosenbrock()
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "Initial guess is not within "
+                                    "the specified bounds")
+            result = minimize(prob.fun, [-10, -10],
+                              method='Nelder-Mead',
+                              bounds=bounds)
+            assert np.less_equal(bounds.lb, result.x).all()
+            assert np.less_equal(result.x, bounds.ub).all()
+            assert np.allclose(prob.fun(result.x), result.fun)
+            assert np.allclose(result.x, x_opt, atol=1.e-3)
+
+    def test_equal_all_bounds(self):
+        prob = Rosenbrock()
+        bounds = Bounds([4.0, 5.0], [4.0, 5.0])
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "Initial guess is not within "
+                                    "the specified bounds")
+            result = minimize(prob.fun, [-10, 8],
+                              method='Nelder-Mead',
+                              bounds=bounds)
+            assert np.allclose(result.x, [4.0, 5.0])
+
+    def test_equal_one_bounds(self):
+        prob = Rosenbrock()
+        bounds = Bounds([4.0, 5.0], [4.0, 20.0])
+        with suppress_warnings() as sup:
+            sup.filter(UserWarning, "Initial guess is not within "
+                                    "the specified bounds")
+            result = minimize(prob.fun, [-10, 8],
+                              method='Nelder-Mead',
+                              bounds=bounds)
+            assert np.allclose(result.x, [4.0, 16.0])
+
+    def test_invalid_bounds(self):
+        prob = Rosenbrock()
+        with raises(ValueError, match=r"one of the lower bounds is greater "
+                                      r"than an upper bound."):
+            bounds = Bounds([-np.inf, 1.0], [4.0, -5.0])
+            minimize(prob.fun, [-10, 3],
+                     method='Nelder-Mead',
+                     bounds=bounds)
+
+    def test_outside_bounds_warning(self):
+        prob = Rosenbrock()
+        with raises(UserWarning, match=r"Initial guess is not within "
+                                       r"the specified bounds"):
+            bounds = Bounds([-np.inf, 1.0], [4.0, 5.0])
+            minimize(prob.fun, [-10, 8],
+                     method='Nelder-Mead',
+                     bounds=bounds)