Repository: scipy/scipy

Issue:
BUG: [issue in scipy.optimize.shgo, for COBYLA's minimizer_kwargs in scipy.optimize._shgo.py]
### Describe your issue.

In file scipy.optimize._shgo.py on line 557 in the declaration of solver_args, the solver method's specific arguments, the 'catol' as one of COBYLA's method specific argument is missing, leading this argument to be removed on line 584:
" _restrict_to_keys(self.minimizer_kwargs['options'],
                          self.min_solver_args + ['ftol'])"
Therefore the catol args is reset to its default value when cobyla is called upon each value of the minimizer pool (same issue with 'rhobeg' argument).

Please fix this in the next patch,
Have a good day.
![issue_coybla_args_in_shgo](https://user-images.githubusercontent.com/67007859/188654988-ca31db08-ec2b-434d-971f-c16e880aa457.png)

See attached example, print the catol value used in cobyla minimizer compared with the 0.05 catol args passed as argument in the minimizer_kwargs of shgo.

### Reproducing Code Example

```python
import numpy as np
from scipy.optimize import shgo


def eggholder(x):
    return (-(x[1] + 47) * np.sin(np.sqrt(abs(x[0]/2 + (x[1]  + 47))))
            -x[0] * np.sin(np.sqrt(abs(x[0] - (x[1]  + 47)))))

bounds = [(-512, 512), (-512, 512)]

def cons(x):
    return x[0] - 0

res_optim = shgo(func=eggholder,
                     bounds=bounds,
                     constraints = {'fun': cons, 'type': 'ineq'},
                     minimizer_kwargs={
                         'method': 'COBYLA',
                         'options': {
                             'rhobeg': 5,
                             'tol': 5e-1,
                             'catol': 0.05
                         }
                     },
                     n=200,
                     iters=5,
                     options={
                         'f_tol': 5e-5,
                         'disp': True
                     })
```


### Error message

```shell
No error
```


### SciPy/NumPy/Python version information

1.8.1 1.23.1 sys.version_info(major=3, minor=10, micro=1, releaselevel='final', serial=0)


Discussion:
Thank you @thibaudrosin. We will see to have this in our big re-write #16313

Resolving patch:
diff --git a/scipy/optimize/_shgo.py b/scipy/optimize/_shgo.py
--- a/scipy/optimize/_shgo.py
+++ b/scipy/optimize/_shgo.py
@@ -11,6 +11,7 @@
 from scipy.optimize import OptimizeResult, minimize, Bounds
 from scipy.optimize._optimize import MemoizeJac
 from scipy.optimize._constraints import new_bounds_to_old
+from scipy.optimize._minimize import standardize_constraints
 from scipy._lib._util import _FunctionWrapper
 
 from scipy.optimize._shgo_lib._complex import Complex
@@ -44,33 +45,15 @@ def shgo(
     args : tuple, optional
         Any additional fixed parameters needed to completely specify the
         objective function.
-    constraints : dict or sequence of dict, optional
-        Constraints definition.
-        Function(s) ``R**n`` in the form::
-
-            g(x) >= 0 applied as g : R^n -> R^m
-            h(x) == 0 applied as h : R^n -> R^p
-
-        Each constraint is defined in a dictionary with fields:
-
-            type : str
-                Constraint type: 'eq' for equality, 'ineq' for inequality.
-            fun : callable
-                The function defining the constraint.
-            jac : callable, optional
-                The Jacobian of `fun` (only for SLSQP).
-            args : sequence, optional
-                Extra arguments to be passed to the function and Jacobian.
-
-        Equality constraint means that the constraint function result is to
-        be zero whereas inequality means that it is to be non-negative.
-        Note that COBYLA only supports inequality constraints.
+    constraints : {Constraint, dict} or List of {Constraint, dict}, optional
+        Constraints definition. Only for COBYLA, SLSQP and trust-constr.
+        See the tutorial [5]_ for further details on specifying constraints.
 
         .. note::
 
-           Only the COBYLA and SLSQP local minimize methods currently
-           support constraint arguments. If the ``constraints`` sequence
-           used in the local optimization problem is not defined in
+           Only COBYLA, SLSQP, and trust-constr local minimize methods
+           currently support constraint arguments. If the ``constraints``
+           sequence used in the local optimization problem is not defined in
            ``minimizer_kwargs`` and a constrained method is used then the
            global ``constraints`` will be used.
            (Defining a ``constraints`` sequence in ``minimizer_kwargs``
@@ -78,6 +61,11 @@ def shgo(
            constraints and so forth need to be added then the inequality
            functions in ``constraints`` need to be added to
            ``minimizer_kwargs`` too).
+           COBYLA only supports inequality constraints.
+
+        .. versionchanged:: 1.11.0
+
+           ``constraints`` accepts `NonlinearConstraint`, `LinearConstraint`.
 
     n : int, optional
         Number of sampling points used in the construction of the simplicial
@@ -226,9 +214,17 @@ def shgo(
         sampling points of dimension ``dim`` per call and output an array of
         sampling points with shape `n x dim`.
 
-    workers : int  optional
-        Uses `multiprocessing.Pool <multiprocessing>`) to sample and run the
-        local serial minimizatons in parrallel.
+    workers : int or map-like callable, optional
+        Sample and run the local serial minimizations in parallel.
+        Supply -1 to use all available CPU cores, or an int to use
+        that many Processes (uses `multiprocessing.Pool <multiprocessing>`).
+
+        Alternatively supply a map-like callable, such as
+        `multiprocessing.Pool.map` for parallel evaluation.
+        This evaluation is carried out as ``workers(func, iterable)``.
+        Requires that `func` be pickleable.
+
+        .. versionadded:: 1.11.0
 
     Returns
     -------
@@ -298,6 +294,7 @@ def shgo(
     .. [4] Wales, DJ (2015) "Perspective: Insight into reaction coordinates and
            dynamics from the potential energy landscape",
            Journal of Chemical Physics, 142(13), 2015.
+    .. [5] https://docs.scipy.org/doc/scipy/tutorial/optimize.html#constrained-minimization-of-multivariate-scalar-functions-minimize
 
     Examples
     --------
@@ -446,14 +443,15 @@ def shgo(
         bounds = new_bounds_to_old(bounds.lb, bounds.ub, len(bounds.lb))
 
     # Initiate SHGO class
-    shc = SHGO(func, bounds, args=args, constraints=constraints, n=n,
+    # use in context manager to make sure that any parallelization
+    # resources are freed.
+    with SHGO(func, bounds, args=args, constraints=constraints, n=n,
                iters=iters, callback=callback,
                minimizer_kwargs=minimizer_kwargs,
                options=options, sampling_method=sampling_method,
-               workers=workers)
-
-    # Run the algorithm, process results and test success
-    shc.iterate_all()
+               workers=workers) as shc:
+        # Run the algorithm, process results and test success
+        shc.iterate_all()
 
     if not shc.break_routine:
         if shc.disp:
@@ -538,12 +536,19 @@ def __init__(self, func, bounds, args=(), constraints=None, n=None,
             self.min_cons = constraints
             self.g_cons = []
             self.g_args = []
-            if (type(constraints) is not tuple) and (type(constraints)
-                                                     is not list):
-                constraints = (constraints,)
 
-            for cons in constraints:
-                if cons['type'] == 'ineq':
+            # shgo internals deals with old-style constraints
+            # self.constraints is used to create Complex, so need
+            # to be stored internally in old-style.
+            # `minimize` takes care of normalising these constraints
+            # for slsqp/cobyla/trust-constr.
+            self.constraints = standardize_constraints(
+                constraints,
+                np.empty(self.dim, float),
+                'old'
+            )
+            for cons in self.constraints:
+                if cons['type'] in ('ineq'):
                     self.g_cons.append(cons['fun'])
                     try:
                         self.g_args.append(cons['args'])
@@ -569,11 +574,15 @@ def __init__(self, func, bounds, args=(), constraints=None, n=None,
         else:
             self.minimizer_kwargs['options'] = {'ftol': 1e-12}
 
-        if (self.minimizer_kwargs['method'] in ('SLSQP', 'COBYLA') and
-                (minimizer_kwargs is not None and
-                 'constraints' not in minimizer_kwargs and
-                 constraints is not None) or
-                (self.g_cons is not None)):
+        if (
+            self.minimizer_kwargs['method'].lower() in ('slsqp', 'cobyla', 'trust-constr') and
+            (
+                minimizer_kwargs is not None and
+                'constraints' not in minimizer_kwargs and
+                constraints is not None
+            ) or
+            (self.g_cons is not None)
+        ):
             self.minimizer_kwargs['constraints'] = self.min_cons
 
         # Process options dict
@@ -615,13 +624,13 @@ def __init__(self, func, bounds, args=(), constraints=None, n=None,
             'newton-cg': ['jac', 'hess', 'hessp'],
             'l-bfgs-b': ['jac', 'bounds'],
             'tnc': ['jac', 'bounds'],
-            'cobyla': ['constraints'],
+            'cobyla': ['constraints', 'catol'],
             'slsqp': ['jac', 'bounds', 'constraints'],
             'dogleg': ['jac', 'hess'],
             'trust-ncg': ['jac', 'hess', 'hessp'],
             'trust-krylov': ['jac', 'hess', 'hessp'],
             'trust-exact': ['jac', 'hess'],
-            'trust-constr': ['jac', 'hess', 'hessp'],
+            'trust-constr': ['jac', 'hess', 'hessp', 'constraints'],
         }
         method = self.minimizer_kwargs['method']
         self.min_solver_args += solver_args[method.lower()]
@@ -795,6 +804,9 @@ def init_options(self, options):
         # Feedback
         self.disp = options.get('disp', False)
 
+    def __enter__(self):
+        return self
+
     def __exit__(self, *args):
         return self.HC.V._mapwrapper.__exit__(*args)
 
@@ -1081,7 +1093,7 @@ def iterate_delaunay(self):
 
         if self.disp:
             logging.info('Triangulation completed, evaluating all '
-                         'contraints and objective function values.')
+                         'constraints and objective function values.')
 
         if hasattr(self, 'Tri'):
             self.HC.vf_to_vv(self.Tri.points, self.Tri.simplices)
@@ -1450,7 +1462,16 @@ def sampling_subspace(self):
         """Find subspace of feasible points from g_func definition"""
         # Subspace of feasible points.
         for ind, g in enumerate(self.g_cons):
-            self.C = self.C[g(self.C.T, *self.g_args[ind]) >= 0.0]
+            # C.shape = (Z, dim) where Z is the number of sampling points to
+            # evaluate and dim is the dimensionality of the problem.
+            # the constraint function may not be vectorised so have to step
+            # through each sampling point sequentially.
+            feasible = np.array(
+                [np.all(g(x_C, *self.g_args[ind]) >= 0.0) for x_C in self.C],
+                dtype=bool
+            )
+            self.C = self.C[feasible]
+
             if self.C.size == 0:
                 self.res.message = ('No sampling point found within the '
                                     + 'feasible set. Increasing sampling '
diff --git a/scipy/optimize/_shgo_lib/_complex.py b/scipy/optimize/_shgo_lib/_complex.py
--- a/scipy/optimize/_shgo_lib/_complex.py
+++ b/scipy/optimize/_shgo_lib/_complex.py
@@ -3,7 +3,7 @@
 import logging
 import itertools
 import decimal
-from functools import lru_cache
+from functools import cache
 
 import numpy
 
@@ -142,7 +142,7 @@ def __init__(self, dim, domain=None, sfield=None, sfield_args=(),
                 constraints = (constraints,)
 
             for cons in constraints:
-                if cons['type'] == 'ineq':
+                if cons['type'] in ('ineq'):
                     self.g_cons.append(cons['fun'])
                     try:
                         self.g_args.append(cons['args'])
@@ -996,7 +996,7 @@ def refine_star(self, v):
                 d_v0v1.connect(d_v1v2)
         return
 
-    @lru_cache(maxsize=None)
+    @cache
     def split_edge(self, v1, v2):
         v1 = self.V[v1]
         v2 = self.V[v2]
diff --git a/scipy/optimize/_shgo_lib/_vertex.py b/scipy/optimize/_shgo_lib/_vertex.py
--- a/scipy/optimize/_shgo_lib/_vertex.py
+++ b/scipy/optimize/_shgo_lib/_vertex.py
@@ -274,17 +274,20 @@ def __init__(self, field=None, field_args=(), g_cons=None, g_cons_args=(),
         self.Vertex = VertexScalarField
         self.field = field
         self.field_args = field_args
-        self.wfield = FieldWraper(field, field_args)  # if workers is not 1
+        self.wfield = FieldWrapper(field, field_args)  # if workers is not 1
 
         self.g_cons = g_cons
         self.g_cons_args = g_cons_args
-        self.wgcons = ConstraintWraper(g_cons, g_cons_args)
+        self.wgcons = ConstraintWrapper(g_cons, g_cons_args)
         self.gpool = set()  # A set of tuples to process for feasibility
 
         # Field processing objects
         self.fpool = set()  # A set of tuples to process for scalar function
         self.sfc_lock = False  # True if self.fpool is non-Empty
 
+        self.workers = workers
+        self._mapwrapper = MapWrapper(workers)
+
         if workers == 1:
             self.process_gpool = self.proc_gpool
             if g_cons is None:
@@ -292,9 +295,6 @@ def __init__(self, field=None, field_args=(), g_cons=None, g_cons_args=(),
             else:
                 self.process_fpool = self.proc_fpool_g
         else:
-            self.workers = workers
-            self._mapwrapper = MapWrapper(workers)
-            self.pool = self._mapwrapper.pool
             self.process_gpool = self.pproc_gpool
             if g_cons is None:
                 self.process_fpool = self.pproc_fpool_nog
@@ -330,7 +330,8 @@ def process_pools(self):
     def feasibility_check(self, v):
         v.feasible = True
         for g, args in zip(self.g_cons, self.g_cons_args):
-            if g(v.x_a, *args) < 0.0:
+            # constraint may return more than 1 value.
+            if np.any(g(v.x_a, *args) < 0.0):
                 v.f = np.inf
                 v.feasible = False
                 break
@@ -365,7 +366,7 @@ def pproc_gpool(self):
         for v in self.gpool:
             gpool_l.append(v.x_a)
 
-        G = self.pool.map(self.wgcons.gcons, gpool_l)
+        G = self._mapwrapper(self.wgcons.gcons, gpool_l)
         for v, g in zip(self.gpool, G):
             v.feasible = g  # set vertex object attribute v.feasible = g (bool)
 
@@ -395,7 +396,7 @@ def pproc_fpool_g(self):
                 fpool_l.append(v.x_a)
             else:
                 v.f = np.inf
-        F = self.pool.map(self.wfield.func, fpool_l)
+        F = self._mapwrapper(self.wfield.func, fpool_l)
         for va, f in zip(fpool_l, F):
             vt = tuple(va)
             self[vt].f = f  # set vertex object attribute v.f = f
@@ -411,7 +412,7 @@ def pproc_fpool_nog(self):
         fpool_l = []
         for v in self.fpool:
             fpool_l.append(v.x_a)
-        F = self.pool.map(self.wfield.func, fpool_l)
+        F = self._mapwrapper(self.wfield.func, fpool_l)
         for va, f in zip(fpool_l, F):
             vt = tuple(va)
             self[vt].f = f  # set vertex object attribute v.f = f
@@ -426,7 +427,7 @@ def proc_minimisers(self):
             v.maximiser()
 
 
-class ConstraintWraper:
+class ConstraintWrapper:
     """Object to wrap constraints to pass to `multiprocessing.Pool`."""
     def __init__(self, g_cons, g_cons_args):
         self.g_cons = g_cons
@@ -435,13 +436,14 @@ def __init__(self, g_cons, g_cons_args):
     def gcons(self, v_x_a):
         vfeasible = True
         for g, args in zip(self.g_cons, self.g_cons_args):
-            if g(v_x_a, *args) < 0.0:
+            # constraint may return more than 1 value.
+            if np.any(g(v_x_a, *args) < 0.0):
                 vfeasible = False
                 break
         return vfeasible
 
 
-class FieldWraper:
+class FieldWrapper:
     """Object to wrap field to pass to `multiprocessing.Pool`."""
     def __init__(self, field, field_args):
         self.field = field


Test patch:
diff --git a/scipy/optimize/tests/test__shgo.py b/scipy/optimize/tests/test__shgo.py
--- a/scipy/optimize/tests/test__shgo.py
+++ b/scipy/optimize/tests/test__shgo.py
@@ -1,11 +1,15 @@
 import logging
+
 import numpy
+import numpy as np
 import time
+from multiprocessing import Pool
 from numpy.testing import assert_allclose
 import pytest
 from pytest import raises as assert_raises, warns
 from scipy.optimize import (shgo, Bounds, minimize_scalar, minimize, rosen,
-                            rosen_der, rosen_hess)
+                            rosen_der, rosen_hess, NonlinearConstraint)
+from scipy.optimize._constraints import new_constraint_to_old
 from scipy.optimize._shgo import SHGO
 
 
@@ -115,6 +119,7 @@ class StructTest3(StructTestFunction):
 
     """
 
+    # amended to test vectorisation of constraints
     def f(self, x):
         return 0.01 * (x[0]) ** 2 + (x[1]) ** 2
 
@@ -124,10 +129,15 @@ def g1(x):
     def g2(x):
         return x[0] ** 2 + x[1] ** 2 - 25.0
 
-    g = (g1, g2)
+    # g = (g1, g2)
+    # cons = wrap_constraints(g)
 
-    cons = wrap_constraints(g)
+    def g(x):
+        return x[0] * x[1] - 25.0, x[0] ** 2 + x[1] ** 2 - 25.0
 
+    # this checks that shgo can be sent new-style constraints
+    __nlc = NonlinearConstraint(g, 0, np.inf)
+    cons = (__nlc,)
 
 test3_1 = StructTest3(bounds=[(2, 50), (0, 50)],
                       expected_x=[250 ** 0.5, 2.5 ** 0.5],
@@ -775,8 +785,12 @@ def f(x):
     def test_19_parallelization(self):
         """Test the functionality to add custom sampling methods to shgo"""
 
-        run_test(test1_1, n=30, workers=1)  # Constrained
-        run_test(test_s, n=30, workers=1)  # Unconstrained
+        with Pool(2) as p:
+            run_test(test1_1, n=30, workers=p.map)  # Constrained
+        run_test(test1_1, n=30, workers=map)  # Constrained
+        with Pool(2) as p:
+            run_test(test_s, n=30, workers=p.map)  # Unconstrained
+        run_test(test_s, n=30, workers=map)  # Unconstrained
 
     def test_20_constrained_args(self):
         """Test that constraints can be passed to arguments"""
@@ -1051,3 +1065,92 @@ def fun(x):
 
         result = shgo(fun, bounds, sampling_method='sobol')
         numpy.testing.assert_equal(fun.nfev, result.nfev)
+
+
+def test_vector_constraint():
+    # gh15514
+    def quad(x):
+        x = np.asarray(x)
+        return [np.sum(x ** 2)]
+
+    nlc = NonlinearConstraint(quad, [2.2], [3])
+    oldc = new_constraint_to_old(nlc, np.array([1.0, 1.0]))
+
+    res = shgo(rosen, [(0, 10), (0, 10)], constraints=oldc, sampling_method='sobol')
+    assert np.all(np.sum((res.x)**2) >= 2.2)
+    assert np.all(np.sum((res.x) ** 2) <= 3.0)
+    assert res.success
+
+
+@pytest.mark.filterwarnings("ignore:delta_grad")
+def test_trust_constr():
+    def quad(x):
+        x = np.asarray(x)
+        return [np.sum(x ** 2)]
+
+    nlc = NonlinearConstraint(quad, [2.6], [3])
+    minimizer_kwargs = {'method': 'trust-constr'}
+    # note that we don't supply the constraints in minimizer_kwargs,
+    # so if the final result obeys the constraints we know that shgo
+    # passed them on to 'trust-constr'
+    res = shgo(
+        rosen,
+        [(0, 10), (0, 10)],
+        constraints=nlc,
+        sampling_method='sobol',
+        minimizer_kwargs=minimizer_kwargs
+    )
+    assert np.all(np.sum((res.x)**2) >= 2.6)
+    assert np.all(np.sum((res.x) ** 2) <= 3.0)
+    assert res.success
+
+
+def test_equality_constraints():
+    # gh16260
+    bounds = [(0.9, 4.0)] * 2  # Constrain probabilities to 0 and 1.
+
+    def faulty(x):
+        return x[0] + x[1]
+
+    nlc = NonlinearConstraint(faulty, 3.9, 3.9)
+    res = shgo(rosen, bounds=bounds, constraints=nlc)
+    assert_allclose(np.sum(res.x), 3.9)
+
+    def faulty(x):
+        return x[0] + x[1] - 3.9
+
+    constraints = {'type': 'eq', 'fun': faulty}
+    res = shgo(rosen, bounds=bounds, constraints=constraints)
+    assert_allclose(np.sum(res.x), 3.9)
+
+    bounds = [(0, 1.0)] * 4
+    # sum of variable should equal 1.
+    def faulty(x):
+        return x[0] + x[1] + x[2] + x[3] - 1
+
+    # options = {'minimize_every_iter': True, 'local_iter':10}
+    constraints = {'type': 'eq', 'fun': faulty}
+    res = shgo(
+        lambda x: - np.prod(x),
+        bounds=bounds,
+        constraints=constraints,
+        sampling_method='sobol'
+    )
+    assert_allclose(np.sum(res.x), 1.0)
+
+def test_gh16971():
+    def cons(x):
+        return np.sum(x**2) - 0
+
+    c = {'fun': cons, 'type': 'ineq'}
+    minimizer_kwargs = {
+        'method': 'COBYLA',
+        'options': {'rhobeg': 5, 'tol': 5e-1, 'catol': 0.05}
+    }
+
+    s = SHGO(
+        rosen, [(0, 10)]*2, constraints=c, minimizer_kwargs=minimizer_kwargs
+    )
+
+    assert s.minimizer_kwargs['method'].lower() == 'cobyla'
+    assert s.minimizer_kwargs['options']['catol'] == 0.05