Repository: scipy/scipy

Issue:
Vector-valued constraints in minimize() et al
minimize() API currently requires that the constraint functions return scalars.

However, SLSQP and COBYLA both call all constraint functions one-by-one, and reconstruct a vector out of the results. This has non-negligible Python overhead if you have many constraints (e.g. matrix-specified constraints of the form `A x = b` or `B x >= 0`).

It should be made possible that you can just give a single constraint function that computes a vector of values `g_i(x)`, `i=0..n-1` at once in the `g_i(x) >= 0`, `i = 0...n-1` inequality or equality constraints.



Discussion:
Isn't it already possible? See `test_minimize_inequality_given_vector_constraints` in `test_slsql.py` which uses a vector-valued inequality constraint `f_ieqcon2`. Or am I missing something?

Hmm, I misread this stackoverflow post:
https://stackoverflow.com/questions/27851087/matrix-constraints-with-scipy-optimize-minimize
.
The more precise claim seems to be that it breaks if you have more than
one such functions.

AFAICT, SLSQP does not break with multiple vector-valued constraints.

About the stackoverflow post, there's arguably something badly defined with the second inequality constraint in `d_` as `b` has shape 2x1 and `A` `2x3` so when `x` (the optimization variable) will get in, it'll have shape `3` (1d array), so the result of this constraint will be `2x2`. Not sure what could be done on our side, but if this user just flattened his `b` array, I'd expect he'd get something correct.

It appears that SLSQP takes in vector constraints but not COBYLA.  When I try to use vector constraints, it throws an ValueError "setting an array element with a sequence" at line 240 in cobyla.py.   


Resolving patch:
diff --git a/scipy/optimize/_minimize.py b/scipy/optimize/_minimize.py
--- a/scipy/optimize/_minimize.py
+++ b/scipy/optimize/_minimize.py
@@ -37,7 +37,7 @@ def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
              hessp=None, bounds=None, constraints=(), tol=None,
              callback=None, options=None):
     """Minimization of scalar function of one or more variables.
-     
+    
     In general, the optimization problems are of the form:
     
     minimize f(x)
@@ -215,7 +215,9 @@ def minimize(fun, x0, args=(), method=None, jac=None, hess=None,
     Constrained Optimization BY Linear Approximation (COBYLA) method
     [9]_, [10]_, [11]_. The algorithm is based on linear
     approximations to the objective function and each constraint. The
-    method wraps a FORTRAN implementation of the algorithm.
+    method wraps a FORTRAN implementation of the algorithm. The
+    constraints functions 'fun' may return either a single number
+    or an array or list of numbers.
 
     Method :ref:`SLSQP <optimize.minimize-slsqp>` uses Sequential
     Least SQuares Programming to minimize a function of several
diff --git a/scipy/optimize/cobyla.py b/scipy/optimize/cobyla.py
--- a/scipy/optimize/cobyla.py
+++ b/scipy/optimize/cobyla.py
@@ -16,6 +16,10 @@
 from scipy._lib.six import callable
 from scipy.optimize import _cobyla
 from .optimize import OptimizeResult, _check_unknown_options
+try:
+    from itertools import izip
+except ImportError:
+    izip = zip
 
 
 __all__ = ['fmin_cobyla']
@@ -37,7 +41,8 @@ def fmin_cobyla(func, x0, cons, args=(), consargs=None, rhobeg=1.0,
     cons : sequence
         Constraint functions; must all be ``>=0`` (a single function
         if only 1 constraint). Each function takes the parameters `x`
-        as its first argument.
+        as its first argument, and it can return either a single number or
+        an array or list of numbers.
     args : tuple, optional
         Extra arguments to pass to function.
     consargs : tuple, optional
@@ -231,12 +236,24 @@ def _minimize_cobyla(fun, x0, args=(), constraints=(),
         if 'args' not in con:
             con['args'] = ()
 
-    m = len(constraints)
+    # m is the total number of constraint values
+    # it takes into account that some constraints may be vector-valued
+    cons_lengths = []
+    for c in constraints:
+        f = c['fun'](x0, *c['args'])
+        try:
+            cons_length = len(f)
+        except TypeError:
+            cons_length = 1
+        cons_lengths.append(cons_length)
+    m = sum(cons_lengths)
 
     def calcfc(x, con):
         f = fun(x, *args)
-        for k, c in enumerate(constraints):
-            con[k] = c['fun'](x, *c['args'])
+        i = 0
+        for size, c in izip(cons_lengths, constraints):
+            con[i: i + size] = c['fun'](x, *c['args'])
+            i += size
         return f
 
     info = np.zeros(4, np.float64)


Test patch:
diff --git a/scipy/optimize/tests/test_cobyla.py b/scipy/optimize/tests/test_cobyla.py
--- a/scipy/optimize/tests/test_cobyla.py
+++ b/scipy/optimize/tests/test_cobyla.py
@@ -11,7 +11,7 @@
 
 class TestCobyla(TestCase):
     def setUp(self):
-        self.x0 = [4.95,0.66]
+        self.x0 = [4.95, 0.66]
         self.solution = [math.sqrt(25 - (2.0/3)**2), 2.0/3]
         self.opts = {'disp': False, 'rhobeg': 1, 'tol': 1e-5,
                      'maxiter': 100}
@@ -72,5 +72,47 @@ def c3(w):
         assert_(not sol.success)
 
 
+def test_vector_constraints():
+    # test that fmin_cobyla and minimize can take a combination
+    # of constraints, some returning a number and others an array
+    def fun(x):
+        return (x[0] - 1)**2 + (x[1] - 2.5)**2
+
+    def fmin(x):
+        return fun(x) - 1
+
+    def cons1(x):
+        a = np.array([[1, -2, 2], [-1, -2, 6], [-1, 2, 2]])
+        return np.array([a[i, 0] * x[0] + a[i, 1] * x[1] +
+                         a[i, 2] for i in range(len(a))])
+
+    def cons2(x):
+        return x     # identity, acts as bounds x > 0
+
+    x0 = np.array([2, 0])
+    cons_list = [fun, cons1, cons2]
+
+    xsol = [1.4, 1.7]
+    fsol = 0.8
+
+    # testing fmin_cobyla
+    sol = fmin_cobyla(fun, x0, cons_list, iprint=0)
+    assert_allclose(sol, xsol, atol=1e-4)
+
+    sol = fmin_cobyla(fun, x0, fmin, iprint=0)
+    assert_allclose(fun(sol), 1, atol=1e-4)
+
+    # testing minimize
+    constraints = [{'type': 'ineq', 'fun': cons} for cons in cons_list]
+    sol = minimize(fun, x0, constraints=constraints)
+    assert_allclose(sol.x, xsol, atol=1e-4)
+    assert_(sol.success, sol.message)
+    assert_allclose(sol.fun, fsol, atol=1e-4)
+
+    constraints = {'type': 'ineq', 'fun': fmin}
+    sol = minimize(fun, x0, constraints=constraints)
+    assert_allclose(sol.fun, 1, atol=1e-4)
+
+
 if __name__ == "__main__":
     run_module_suite()