Repository: numba/numba

Issue:
Allow multiple outputs for `guvectorize` on CUDA target
## Feature request

The following works just fine for targets `cpu` and `parallel` ...

```python
from numba import guvectorize
import numpy as np

@guvectorize(
   'void(f8[:],f8[:],f8[:],f8[:],f8[:],f8[:])',
    '(n),(n),(n)->(n),(n),(n)',
    target = 'cpu',
    nopython = True,
)
def foo(a, b, c, x, y, z):
    for idx in range(a.shape[0]):
        x[idx], y[idx], z[idx] = a[idx] * b[idx], b[idx] * c[idx], c[idx] * a[idx]

LEN = 100_000_000

data = np.arange(0, 3 * LEN, dtype = 'f8').reshape(3, LEN)
res = np.zeros_like(data)

foo(data[0], data[1], data[2], res[0], res[1], res[2])  # providing views on `res`

print(res)
```

... but fails to compile with `AssertionError: only support 1 output` for target `cuda`. It would be *really* useful if this would also work for CUDA.

In the meantime, the current behavior is, as far as I can tell, not documented.

---

For a longer discussion, context and examples, see [here](https://numba.discourse.group/t/returning-an-array-of-3d-or-6d-vectors-from-guvectorize-on-cuda-cross-compiling-for-cpu-cuda/1473).


Discussion:
Hi @s-m-e, thanks for the report. I'll put this up for discussion at the next triage meeting.
@s-m-e I'm starting working on this on this branch: https://github.com/gmarkall/numba/tree/issue-8303

Presently I'm just hacking away to see what needs changing ("brute force" modifications :slightly_smiling_face:) because I'm not too familiar with the code in this area. Hopefully it will just need a bunch of small fixes to turn the assumption of a single output into iterations over multiple outputs in various locations.

Resolving patch:
diff --git a/numba/cuda/vectorizers.py b/numba/cuda/vectorizers.py
--- a/numba/cuda/vectorizers.py
+++ b/numba/cuda/vectorizers.py
@@ -12,17 +12,8 @@ class CUDAUFuncDispatcher(object):
 
     def __init__(self, types_to_retty_kernels, pyfunc):
         self.functions = types_to_retty_kernels
-        self._maxblocksize = 0  # ignored
         self.__name__ = pyfunc.__name__
 
-    @property
-    def max_blocksize(self):
-        return self._maxblocksize
-
-    @max_blocksize.setter
-    def max_blocksize(self, blksz):
-        self._max_blocksize = blksz
-
     def __call__(self, *args, **kws):
         """
         *args: numpy arrays or DeviceArrayBase (created by cuda.to_device).
@@ -94,6 +85,10 @@ class _CUDAGUFuncCallSteps(GUFuncCallSteps):
         '_stream',
     ]
 
+    def __init__(self, nin, nout, args, kwargs):
+        super().__init__(nin, nout, args, kwargs)
+        self._stream = kwargs.get('stream', 0)
+
     def is_device_array(self, obj):
         return cuda.is_cuda_array(obj)
 
@@ -114,12 +109,9 @@ def to_host(self, devary, hostary):
         out = devary.copy_to_host(hostary, stream=self._stream)
         return out
 
-    def device_array(self, shape, dtype):
+    def allocate_device_array(self, shape, dtype):
         return cuda.device_array(shape=shape, dtype=dtype, stream=self._stream)
 
-    def prepare_inputs(self):
-        self._stream = self.kwargs.get('stream', 0)
-
     def launch_kernel(self, kernel, nelem, args):
         kernel.forall(nelem, stream=self._stream)(*args)
 
@@ -177,7 +169,7 @@ def to_device(self, hostary, stream):
     def to_host(self, devary, stream):
         return devary.copy_to_host(stream=stream)
 
-    def device_array(self, shape, dtype, stream):
+    def allocate_device_array(self, shape, dtype, stream):
         return cuda.device_array(shape=shape, dtype=dtype, stream=stream)
 
     def broadcast_device(self, ary, shape):
diff --git a/numba/np/ufunc/deviceufunc.py b/numba/np/ufunc/deviceufunc.py
--- a/numba/np/ufunc/deviceufunc.py
+++ b/numba/np/ufunc/deviceufunc.py
@@ -2,6 +2,7 @@
 Implements custom ufunc dispatch mechanism for non-CPU devices.
 """
 
+from abc import ABCMeta, abstractmethod
 from collections import OrderedDict
 import operator
 import warnings
@@ -291,7 +292,7 @@ def attempt_ravel(a):
         shape = args[0].shape
         if out is None:
             # No output is provided
-            devout = cr.device_array(shape, resty, stream=stream)
+            devout = cr.allocate_device_array(shape, resty, stream=stream)
 
             devarys.extend([devout])
             cr.launch(func, shape[0], stream, devarys)
@@ -319,7 +320,7 @@ def attempt_ravel(a):
             # Return host array
             assert out.shape == shape
             assert out.dtype == resty
-            devout = cr.device_array(shape, resty, stream=stream)
+            devout = cr.allocate_device_array(shape, resty, stream=stream)
             devarys.extend([devout])
             cr.launch(func, shape[0], stream, devarys)
             return devout.copy_to_host(out, stream=stream).reshape(outshape)
@@ -336,7 +337,7 @@ def to_host(self, devary, stream):
         """
         raise NotImplementedError
 
-    def device_array(self, shape, dtype, stream):
+    def allocate_device_array(self, shape, dtype, stream):
         """Implements device allocation
         Override in subclass
         """
@@ -376,19 +377,7 @@ def __init__(self, func, identity=None, cache=False, targetoptions={}):
     def pyfunc(self):
         return self.py_func
 
-    def add(self, sig=None, argtypes=None, restype=None):
-        # Handle argtypes
-        if argtypes is not None:
-            warnings.warn("Keyword argument argtypes is deprecated",
-                          DeprecationWarning)
-            assert sig is None
-            if restype is None:
-                sig = tuple(argtypes)
-            else:
-                sig = restype(*argtypes)
-        del argtypes
-        del restype
-
+    def add(self, sig=None):
         # compile core as device function
         args, return_type = sigutils.normalize_signature(sig)
         devfnsig = signature(return_type, *args)
@@ -445,31 +434,27 @@ def __init__(self, func, sig, identity=None, cache=False, targetoptions={}):
         self.identity = parse_identity(identity)
         self.signature = sig
         self.inputsig, self.outputsig = parse_signature(self.signature)
-        assert len(self.outputsig) == 1, "only support 1 output"
-        # { arg_dtype: (return_dtype), cudakernel }
+
+        # Maps from a tuple of input_dtypes to (output_dtypes, kernel)
         self.kernelmap = OrderedDict()
 
     @property
     def pyfunc(self):
         return self.py_func
 
-    def add(self, sig=None, argtypes=None, restype=None):
-        # Handle argtypes
-        if argtypes is not None:
-            warnings.warn("Keyword argument argtypes is deprecated",
-                          DeprecationWarning)
-            assert sig is None
-            if restype is None:
-                sig = tuple(argtypes)
-            else:
-                sig = restype(*argtypes)
-        del argtypes
-        del restype
-
+    def add(self, sig=None):
         indims = [len(x) for x in self.inputsig]
         outdims = [len(x) for x in self.outputsig]
         args, return_type = sigutils.normalize_signature(sig)
 
+        # It is only valid to specify types.none as a return type, or to not
+        # specify the return type (where the "Python None" is the return type)
+        valid_return_type = return_type in (types.none, None)
+        if not valid_return_type:
+            raise TypeError('guvectorized functions cannot return values: '
+                            f'signature {sig} specifies {return_type} return '
+                           'type')
+
         funcname = self.py_func.__name__
         src = expand_gufunc_template(self._kernel_template, indims,
                                      outdims, funcname, args)
@@ -482,8 +467,12 @@ def add(self, sig=None, argtypes=None, restype=None):
         outertys = list(_determine_gufunc_outer_types(args, indims + outdims))
         kernel = self._compile_kernel(fnobj, sig=tuple(outertys))
 
-        dtypes = tuple(np.dtype(str(t.dtype)) for t in outertys)
-        self.kernelmap[tuple(dtypes[:-1])] = dtypes[-1], kernel
+        nout = len(outdims)
+        dtypes = [np.dtype(str(t.dtype)) for t in outertys]
+        indtypes = tuple(dtypes[:-nout])
+        outdtypes = tuple(dtypes[-nout:])
+
+        self.kernelmap[indtypes] = outdtypes, kernel
 
     def _compile_kernel(self, fnobj, sig):
         raise NotImplementedError
@@ -640,44 +629,44 @@ def __init__(self, kernelmap, engine):
         self.kernelmap = kernelmap
         self.engine = engine
         self.max_blocksize = 2 ** 30
-        assert self.engine.nout == 1, "only support single output"
 
     def __call__(self, *args, **kws):
         callsteps = self._call_steps(self.engine.nin, self.engine.nout,
                                      args, kws)
-        callsteps.prepare_inputs()
-        indtypes, schedule, outdtype, kernel = self._schedule(
-            callsteps.norm_inputs, callsteps.output)
+        indtypes, schedule, outdtypes, kernel = self._schedule(
+            callsteps.inputs, callsteps.outputs)
         callsteps.adjust_input_types(indtypes)
-        callsteps.allocate_outputs(schedule, outdtype)
-        callsteps.prepare_kernel_parameters()
-        newparams, newretval = self._broadcast(schedule,
-                                               callsteps.kernel_parameters,
-                                               callsteps.kernel_returnvalue)
-        callsteps.launch_kernel(kernel, schedule.loopn, newparams + [newretval])
-        return callsteps.post_process_result()
-
-    def _schedule(self, inputs, out):
+
+        outputs = callsteps.prepare_outputs(schedule, outdtypes)
+        inputs = callsteps.prepare_inputs()
+        parameters = self._broadcast(schedule, inputs, outputs)
+
+        callsteps.launch_kernel(kernel, schedule.loopn, parameters)
+
+        return callsteps.post_process_outputs(outputs)
+
+    def _schedule(self, inputs, outs):
         input_shapes = [a.shape for a in inputs]
         schedule = self.engine.schedule(input_shapes)
 
         # find kernel
-        idtypes = tuple(i.dtype for i in inputs)
+        indtypes = tuple(i.dtype for i in inputs)
         try:
-            outdtype, kernel = self.kernelmap[idtypes]
+            outdtypes, kernel = self.kernelmap[indtypes]
         except KeyError:
             # No exact match, then use the first compatible.
             # This does not match the numpy dispatching exactly.
             # Later, we may just jit a new version for the missing signature.
-            idtypes = self._search_matching_signature(idtypes)
+            indtypes = self._search_matching_signature(indtypes)
             # Select kernel
-            outdtype, kernel = self.kernelmap[idtypes]
+            outdtypes, kernel = self.kernelmap[indtypes]
 
         # check output
-        if out is not None and schedule.output_shapes[0] != out.shape:
-            raise ValueError('output shape mismatch')
+        for sched_shape, out in zip(schedule.output_shapes, outs):
+            if out is not None and sched_shape != out.shape:
+                raise ValueError('output shape mismatch')
 
-        return idtypes, schedule, outdtype, kernel
+        return indtypes, schedule, outdtypes, kernel
 
     def _search_matching_signature(self, idtypes):
         """
@@ -693,7 +682,7 @@ def _search_matching_signature(self, idtypes):
         else:
             raise TypeError("no matching signature")
 
-    def _broadcast(self, schedule, params, retval):
+    def _broadcast(self, schedule, params, retvals):
         assert schedule.loopn > 0, "zero looping dimension"
 
         odim = 1 if not schedule.loopdims else schedule.loopn
@@ -706,8 +695,11 @@ def _broadcast(self, schedule, params, retval):
             else:
                 # Broadcast vector input
                 newparams.append(self._broadcast_array(p, odim, cs))
-        newretval = retval.reshape(odim, *schedule.oshapes[0])
-        return newparams, newretval
+
+        newretvals = []
+        for retval, oshape in zip(retvals, schedule.oshapes):
+            newretvals.append(retval.reshape(odim, *oshape))
+        return tuple(newparams) + tuple(newretvals)
 
     def _broadcast_array(self, ary, newdim, innerdim):
         newshape = (newdim,) + innerdim
@@ -732,110 +724,177 @@ def _broadcast_scalar_input(self, ary, shape):
         raise NotImplementedError
 
 
-class GUFuncCallSteps(object):
+class GUFuncCallSteps(metaclass=ABCMeta):
+    """
+    Implements memory management and kernel launch operations for GUFunc calls.
+
+    One instance of this class is instantiated for each call, and the instance
+    is specific to the arguments given to the GUFunc call.
+
+    The base class implements the overall logic; subclasses provide
+    target-specific implementations of individual functions.
+    """
+
+    # The base class uses these slots; subclasses may provide additional slots.
     __slots__ = [
-        'args',
-        'kwargs',
-        'output',
-        'norm_inputs',
-        'kernel_returnvalue',
-        'kernel_parameters',
-        '_is_device_array',
-        '_need_device_conversion',
+        'outputs',
+        'inputs',
+        '_copy_result_to_host',
     ]
 
+    @abstractmethod
+    def launch_kernel(self, kernel, nelem, args):
+        """Implement the kernel launch"""
+
+    @abstractmethod
+    def is_device_array(self, obj):
+        """
+        Return True if `obj` is a device array for this target, False
+        otherwise.
+        """
+
+    @abstractmethod
+    def as_device_array(self, obj):
+        """
+        Return `obj` as a device array on this target.
+
+        May return `obj` directly if it is already on the target.
+        """
+
+    @abstractmethod
+    def to_device(self, hostary):
+        """
+        Copy `hostary` to the device and return the device array.
+        """
+
+    @abstractmethod
+    def allocate_device_array(self, shape, dtype):
+        """
+        Allocate a new uninitialized device array with the given shape and
+        dtype.
+        """
+
     def __init__(self, nin, nout, args, kwargs):
-        if nout > 1:
-            raise ValueError('multiple output is not supported')
-        self.args = args
-        self.kwargs = kwargs
-
-        user_output_is_device = False
-        self.output = self.kwargs.get('out')
-        if self.output is not None:
-            user_output_is_device = self.is_device_array(self.output)
-            if user_output_is_device:
-                self.output = self.as_device_array(self.output)
-        self._is_device_array = [self.is_device_array(a) for a in self.args]
-        self._need_device_conversion = (not any(self._is_device_array) and
-                                        not user_output_is_device)
-
-        # Normalize inputs
-        inputs = []
-        for a, isdev in zip(self.args, self._is_device_array):
-            if isdev:
-                inputs.append(self.as_device_array(a))
+        outputs = kwargs.get('out')
+
+        # Ensure the user has passed a correct number of arguments
+        if outputs is None and len(args) not in (nin, (nin + nout)):
+            def pos_argn(n):
+                return f'{n} positional argument{"s" * (n != 1)}'
+
+            msg = (f'This gufunc accepts {pos_argn(nin)} (when providing '
+                   f'input only) or {pos_argn(nin + nout)} (when providing '
+                   f'input and output). Got {pos_argn(len(args))}.')
+            raise TypeError(msg)
+
+        if outputs is not None and len(args) > nin:
+            raise ValueError("cannot specify argument 'out' as both positional "
+                             "and keyword")
+        else:
+            # If the user did not pass outputs either in the out kwarg or as
+            # positional arguments, then we need to generate an initial list of
+            # "placeholder" outputs using None as a sentry value
+            outputs = [outputs] * nout
+
+        # Ensure all output device arrays are Numba device arrays - for
+        # example, any output passed in that supports the CUDA Array Interface
+        # is converted to a Numba CUDA device array; others are left untouched.
+        all_user_outputs_are_host = True
+        self.outputs = []
+        for output in outputs:
+            if self.is_device_array(output):
+                self.outputs.append(self.as_device_array(output))
+                all_user_outputs_are_host = False
+            else:
+                self.outputs.append(output)
+
+        all_host_arrays = not any([self.is_device_array(a) for a in args])
+
+        # - If any of the arguments are device arrays, we leave the output on
+        #   the device.
+        self._copy_result_to_host = (all_host_arrays and
+                                     all_user_outputs_are_host)
+
+        # Normalize arguments - ensure they are either device- or host-side
+        # arrays (as opposed to lists, tuples, etc).
+        def normalize_arg(a):
+            if self.is_device_array(a):
+                convert = self.as_device_array
             else:
-                inputs.append(np.asarray(a))
-        self.norm_inputs = inputs[:nin]
+                convert = np.asarray
+
+            return convert(a)
+
+        normalized_args = [normalize_arg(a) for a in args]
+        self.inputs = normalized_args[:nin]
+
         # Check if there are extra arguments for outputs.
-        unused_inputs = inputs[nin:]
+        unused_inputs = normalized_args[nin:]
         if unused_inputs:
-            if self.output is not None:
-                raise ValueError("cannot specify 'out' as both a positional "
-                                 "and keyword argument")
-            else:
-                [self.output] = unused_inputs
+            self.outputs = unused_inputs
 
     def adjust_input_types(self, indtypes):
         """
         Attempt to cast the inputs to the required types if necessary
-        and if they are not device array.
+        and if they are not device arrays.
 
-        Side effect: Only affects the element of `norm_inputs` that requires
+        Side effect: Only affects the elements of `inputs` that require
         a type cast.
         """
-        for i, (ity, val) in enumerate(zip(indtypes, self.norm_inputs)):
+        for i, (ity, val) in enumerate(zip(indtypes, self.inputs)):
             if ity != val.dtype:
                 if not hasattr(val, 'astype'):
                     msg = ("compatible signature is possible by casting but "
                            "{0} does not support .astype()").format(type(val))
                     raise TypeError(msg)
                 # Cast types
-                self.norm_inputs[i] = val.astype(ity)
+                self.inputs[i] = val.astype(ity)
 
-    def allocate_outputs(self, schedule, outdtype):
-        # allocate output
-        if self._need_device_conversion or self.output is None:
-            retval = self.device_array(shape=schedule.output_shapes[0],
-                                       dtype=outdtype)
-        else:
-            retval = self.output
-        self.kernel_returnvalue = retval
-
-    def prepare_kernel_parameters(self):
-        params = []
-        for inp, isdev in zip(self.norm_inputs, self._is_device_array):
-            if isdev:
-                params.append(inp)
-            else:
-                params.append(self.to_device(inp))
-        assert all(self.is_device_array(a) for a in params)
-        self.kernel_parameters = params
-
-    def post_process_result(self):
-        if self._need_device_conversion:
-            out = self.to_host(self.kernel_returnvalue, self.output)
-        elif self.output is None:
-            out = self.kernel_returnvalue
-        else:
-            out = self.output
-        return out
+    def prepare_outputs(self, schedule, outdtypes):
+        """
+        Returns a list of output parameters that all reside on the target device.
 
-    def prepare_inputs(self):
-        pass
+        Outputs that were passed-in to the GUFunc are used if they reside on the
+        device; other outputs are allocated as necessary.
+        """
+        outputs = []
+        for shape, dtype, output in zip(schedule.output_shapes, outdtypes,
+                                        self.outputs):
+            if output is None or self._copy_result_to_host:
+                output = self.allocate_device_array(shape, dtype)
+            outputs.append(output)
 
-    def launch_kernel(self, kernel, nelem, args):
-        raise NotImplementedError
+        return outputs
 
-    def is_device_array(self, obj):
-        raise NotImplementedError
+    def prepare_inputs(self):
+        """
+        Returns a list of input parameters that all reside on the target device.
+        """
+        def ensure_device(parameter):
+            if self.is_device_array(parameter):
+                convert = self.as_device_array
+            else:
+                convert = self.to_device
 
-    def as_device_array(self, obj):
-        return obj
+            return convert(parameter)
 
-    def to_device(self, hostary):
-        raise NotImplementedError
+        return [ensure_device(p) for p in self.inputs]
 
-    def device_array(self, shape, dtype):
-        raise NotImplementedError
+    def post_process_outputs(self, outputs):
+        """
+        Moves the given output(s) to the host if necessary.
+
+        Returns a single value (e.g. an array) if there was one output, or a
+        tuple of arrays if there were multiple. Although this feels a little
+        jarring, it is consistent with the behavior of GUFuncs in general.
+        """
+        if self._copy_result_to_host:
+            outputs = [self.to_host(output, self_output)
+                       for output, self_output in zip(outputs, self.outputs)]
+        elif self.outputs[0] is not None:
+            outputs = self.outputs
+
+        if len(outputs) == 1:
+            return outputs[0]
+        else:
+            return tuple(outputs)


Test patch:
diff --git a/numba/cuda/tests/cudapy/test_gufunc.py b/numba/cuda/tests/cudapy/test_gufunc.py
--- a/numba/cuda/tests/cudapy/test_gufunc.py
+++ b/numba/cuda/tests/cudapy/test_gufunc.py
@@ -2,7 +2,7 @@
 import numpy.core.umath_tests as ut
 
 from collections import namedtuple
-from numba import void, float32, float64
+from numba import void, int32, float32, float64
 from numba import guvectorize
 from numba import cuda
 from numba.cuda.testing import skip_on_cudasim, CUDATestCase
@@ -12,7 +12,7 @@
 from numba.tests.support import override_config
 
 
-def _get_matmulcore_gufunc(dtype=float32, max_blocksize=None):
+def _get_matmulcore_gufunc(dtype=float32):
     @guvectorize([void(dtype[:, :], dtype[:, :], dtype[:, :])],
                  '(m,n),(n,p)->(m,p)',
                  target='cuda')
@@ -25,10 +25,7 @@ def matmulcore(A, B, C):
                 for k in range(n):
                     C[i, j] += A[i, k] * B[k, j]
 
-    gufunc = matmulcore
-    if max_blocksize:
-        gufunc.max_blocksize = max_blocksize
-    return gufunc
+    return matmulcore
 
 
 @skip_on_cudasim('ufunc API unsupported in the simulator')
@@ -36,7 +33,7 @@ class TestCUDAGufunc(CUDATestCase):
 
     def test_gufunc_small(self):
 
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
 
         matrix_ct = 2
         A = np.arange(matrix_ct * 2 * 4, dtype=np.float32).reshape(matrix_ct, 2,
@@ -50,7 +47,7 @@ def test_gufunc_small(self):
 
     def test_gufunc_auto_transfer(self):
 
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
 
         matrix_ct = 2
         A = np.arange(matrix_ct * 2 * 4, dtype=np.float32).reshape(matrix_ct, 2,
@@ -66,7 +63,7 @@ def test_gufunc_auto_transfer(self):
 
     def test_gufunc(self):
 
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
 
         matrix_ct = 1001 # an odd number to test thread/block division in CUDA
         A = np.arange(matrix_ct * 2 * 4, dtype=np.float32).reshape(matrix_ct, 2,
@@ -80,7 +77,7 @@ def test_gufunc(self):
 
     def test_gufunc_hidim(self):
 
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
 
         matrix_ct = 100 # an odd number to test thread/block division in CUDA
         A = np.arange(matrix_ct * 2 * 4, dtype=np.float32).reshape(4, 25, 2, 4)
@@ -105,24 +102,9 @@ def test_gufunc_new_axis(self):
         res2 = gufunc(X, np.tile(Y, (10, 1, 1)))
         np.testing.assert_allclose(gold, res2)
 
-    def test_gufunc_adjust_blocksize(self):
-
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
-
-        matrix_ct = 1001 # an odd number to test thread/block division in CUDA
-        A = np.arange(matrix_ct * 2 * 4, dtype=np.float32).reshape(matrix_ct, 2,
-                                                                   4)
-        B = np.arange(matrix_ct * 4 * 5, dtype=np.float32).reshape(matrix_ct, 4,
-                                                                   5)
-
-        gufunc.max_blocksize = 32
-        C = gufunc(A, B)
-        Gold = ut.matrix_multiply(A, B)
-        self.assertTrue(np.allclose(C, Gold))
-
     def test_gufunc_stream(self):
 
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
 
         #cuda.driver.flush_pending_free()
         matrix_ct = 1001 # an odd number to test thread/block division in CUDA
@@ -153,6 +135,21 @@ def copy(A, B):
             for i in range(B.size):
                 B[i] = A[i]
 
+        A = np.arange(10, dtype=np.float32) + 1
+        B = np.zeros_like(A)
+        copy(A, out=B)
+        np.testing.assert_allclose(A, B)
+
+    def test_copy_unspecified_return(self):
+        # Ensure that behaviour is correct when the return type is not
+        # specified in the signature.
+        @guvectorize([(float32[:], float32[:])],
+                     '(x)->(x)',
+                     target='cuda')
+        def copy(A, B):
+            for i in range(B.size):
+                B[i] = A[i]
+
         A = np.arange(10, dtype=np.float32) + 1
         B = np.zeros_like(A)
         copy(A, out=B)
@@ -266,7 +263,7 @@ def foo(inp, out):
         with self.assertRaises(ValueError) as raises:
             foo(inp, out, out=out)
 
-        msg = "cannot specify 'out' as both a positional and keyword argument"
+        msg = "cannot specify argument 'out' as both positional and keyword"
         self.assertEqual(str(raises.exception), msg)
 
     def check_tuple_arg(self, a, b):
@@ -305,9 +302,138 @@ def test_tuple_of_array_arg(self):
         self.check_tuple_arg(a, b)
 
     def test_gufunc_name(self):
-        gufunc = _get_matmulcore_gufunc(max_blocksize=512)
+        gufunc = _get_matmulcore_gufunc()
         self.assertEqual(gufunc.__name__, 'matmulcore')
 
+    def test_bad_return_type(self):
+        with self.assertRaises(TypeError) as te:
+            @guvectorize([int32(int32[:], int32[:])], '(m)->(m)', target='cuda')
+            def f(x, y):
+                pass
+
+        msg = str(te.exception)
+        self.assertIn('guvectorized functions cannot return values', msg)
+        self.assertIn('specifies int32 return type', msg)
+
+    def test_incorrect_number_of_pos_args(self):
+        @guvectorize([(int32[:], int32[:], int32[:])],
+                     '(m),(m)->(m)', target='cuda')
+        def f(x, y, z):
+            pass
+
+        arr = np.arange(5)
+
+        # Inputs only, too few
+        with self.assertRaises(TypeError) as te:
+            f(arr)
+
+        msg = str(te.exception)
+        self.assertIn('gufunc accepts 2 positional arguments', msg)
+        self.assertIn('or 3 positional arguments', msg)
+        self.assertIn('Got 1 positional argument.', msg)
+
+        # Inputs and outputs, too many
+        with self.assertRaises(TypeError) as te:
+            f(arr, arr, arr, arr)
+
+        msg = str(te.exception)
+        self.assertIn('gufunc accepts 2 positional arguments', msg)
+        self.assertIn('or 3 positional arguments', msg)
+        self.assertIn('Got 4 positional arguments.', msg)
+
+
+@skip_on_cudasim('ufunc API unsupported in the simulator')
+class TestMultipleOutputs(CUDATestCase):
+    def test_multiple_outputs_same_type_passed_in(self):
+        @guvectorize([void(float32[:], float32[:], float32[:])],
+                     '(x)->(x),(x)',
+                     target='cuda')
+        def copy(A, B, C):
+            for i in range(B.size):
+                B[i] = A[i]
+                C[i] = A[i]
+
+        A = np.arange(10, dtype=np.float32) + 1
+        B = np.zeros_like(A)
+        C = np.zeros_like(A)
+        copy(A, B, C)
+        np.testing.assert_allclose(A, B)
+        np.testing.assert_allclose(A, C)
+
+    def test_multiple_outputs_distinct_values(self):
+
+        @guvectorize([void(float32[:], float32[:], float32[:])],
+                     '(x)->(x),(x)',
+                     target='cuda')
+        def copy_and_double(A, B, C):
+            for i in range(B.size):
+                B[i] = A[i]
+                C[i] = A[i] * 2
+
+        A = np.arange(10, dtype=np.float32) + 1
+        B = np.zeros_like(A)
+        C = np.zeros_like(A)
+        copy_and_double(A, B, C)
+        np.testing.assert_allclose(A, B)
+        np.testing.assert_allclose(A * 2, C)
+
+    def test_multiple_output_allocation(self):
+        @guvectorize([void(float32[:], float32[:], float32[:])],
+                     '(x)->(x),(x)',
+                     target='cuda')
+        def copy_and_double(A, B, C):
+            for i in range(B.size):
+                B[i] = A[i]
+                C[i] = A[i] * 2
+
+        A = np.arange(10, dtype=np.float32) + 1
+        B, C = copy_and_double(A)
+        np.testing.assert_allclose(A, B)
+        np.testing.assert_allclose(A * 2, C)
+
+    def test_multiple_output_dtypes(self):
+
+        @guvectorize([void(int32[:], int32[:], float64[:])],
+                     '(x)->(x),(x)',
+                     target='cuda')
+        def copy_and_multiply(A, B, C):
+            for i in range(B.size):
+                B[i] = A[i]
+                C[i] = A[i] * 1.5
+
+        A = np.arange(10, dtype=np.int32) + 1
+        B = np.zeros_like(A)
+        C = np.zeros_like(A, dtype=np.float64)
+        copy_and_multiply(A, B, C)
+        np.testing.assert_allclose(A, B)
+        np.testing.assert_allclose(A * np.float64(1.5), C)
+
+    def test_incorrect_number_of_pos_args(self):
+        @guvectorize([(int32[:], int32[:], int32[:], int32[:])],
+                     '(m),(m)->(m),(m)', target='cuda')
+        def f(x, y, z, w):
+            pass
+
+        arr = np.arange(5)
+
+        # Inputs only, too few
+        with self.assertRaises(TypeError) as te:
+            f(arr)
+
+        msg = str(te.exception)
+        self.assertIn('gufunc accepts 2 positional arguments', msg)
+        self.assertIn('or 4 positional arguments', msg)
+        self.assertIn('Got 1 positional argument.', msg)
+
+        # Inputs and outputs, too many
+        with self.assertRaises(TypeError) as te:
+            f(arr, arr, arr, arr, arr)
+
+        msg = str(te.exception)
+        self.assertIn('gufunc accepts 2 positional arguments', msg)
+        self.assertIn('or 4 positional arguments', msg)
+        self.assertIn('Got 5 positional arguments.', msg)
+
 
 if __name__ == '__main__':
     unittest.main()