Repository: numba/numba

Issue:
Support gufuncs with no output?
I'd like to be able to write gufuncs with no output variables. I think this is technically valid according to the [gufunc spec](http://docs.scipy.org/doc/numpy/reference/c-api.generalized-ufuncs.html#details-of-signature), although I have not found any examples.

As an example application, this would let me write a gufunc to the core work of a grouped sum (e.g., as found in pandas):

``` python
from numba import guvectorize
import numpy as np
import pandas as pd

@guvectorize(['void(float64[:], float64[:], float64[:])'],
             '(x),(x),(y)->')
def _grouped_sum(values, labels, target):
    for i in range(len(values)):
        target[labels[i]] += values[i]

def grouped_sum(values, by):
    labels, uniques = pd.factorize(by)
    target = np.zeros(uniques.shape)
    _grouped_sum(values, labels, target)
    return target

values = np.random.randn(int(1e4))
by = np.random.randint(10, size=int(1e4))
grouped_sum(values, by)
```

In this case, `target` can't be the output from the gufunc, because the size of the target dimension needs to be computed from the input.

Attempting to execute this currently gives an error:

```
ValueError                                Traceback (most recent call last)
<ipython-input-17-bdd3cb66bdcf> in <module>()
      3 
      4 @guvectorize(['void(float64[:], float64[:], float64[:])'],
----> 5              '(x),(x),(y)->')
      6 def _grouped_sum(values, labels, target):
      7     for i in range(len(values)):

/Users/shoyer/miniconda/envs/numbagg-dev/lib/python2.7/site-packages/numba/npyufunc/decorators.pyc in wrap(func)
    117         for fty in ftylist:
    118             guvec.add(fty)
--> 119         return guvec.build_ufunc()
    120 
    121     return wrap

/Users/shoyer/miniconda/envs/numbagg-dev/lib/python2.7/site-packages/numba/npyufunc/ufuncbuilder.pyc in build_ufunc(self)
    224         # Pass envs to fromfuncsig to bind to the lifetime of the ufunc object
    225         ufunc = _internal.fromfuncsig(ptrlist, dtypelist, inct, outct, datlist,
--> 226                                       self.signature, keepalive)
    227         return ufunc
    228 

ValueError: (x),(x),(y)->
```

I can fix this by adding a dummy output dimension, but that needlessly slows things down:

``` python
@guvectorize(['void(float64[:], float64[:], float64[:], float64[:])'],
             '(x),(x),(y)->()')
def _grouped_sum(values, labels, target, out):
    for i in range(len(values)):
        target[labels[i]] += values[i]
```

CC @jsphon https://github.com/pydata/pandas/issues/8881



Discussion:
Interesting: when I run this example on numba master, I get a segmentation fault and hard crash instead of an error:
/Users/shoyer/miniconda/envs/numba-dev/bin/python.app: line 3: 93218 Segmentation fault: 11  /Users/shoyer/miniconda/envs/numba-dev/python.app/Contents/MacOS/python "$@"

Which example? The one with an output or the one without?

The one without the output causing the segmentation fault on master -- literally exactly the code in my first example pasted into ipython.

I don't think writing to input is valid.  Inputs to the gufunc core can be broadcasted temporaries.

@sklam You do have a good point (in general), though in this case I'm not exposing the gufunc externally, so I can ensure that `target` does not need to be broadcast, i.e., I'll always supply `target = np.zeros(values.shape[:-1] + (n_unique,))`, so target has the same dimensionality as values.

Does this need to be a `@guvectorize` function at all, or could it be a simple `@jit` function?

@seibert The guvectorize version is nice because I can write a single version that works on data of any dimensionality, e.g.,

``` python
from numba import guvectorize, jit
import numpy as np
import pandas as pd

@guvectorize(['void(float64[:], int64[:], float64[:], float64[:])'],
             '(x),(x),(y)->()')
def _grouped_sum_gufunc(values, labels, target, out):
    for i in range(len(values)):
        idx = labels[i]
        target[idx] += values[i]

def grouped_sum_last_axis(values, by):
    labels, uniques = pd.factorize(by, sort=True)
    target = np.zeros(values.shape[:-1] + (uniques.size,), values.dtype)
    _grouped_sum_gufunc(values, labels, target)
    return target

values = np.random.randn(3, 100)
labels = np.random.randint(4, size=100)
grouped_sum_last_axis(values, labels)
```

With an array transpose, I could make grouped_sum work along any axis.

The main limitation of this approach is the inability to create temporary arrays means that I can't use this for grouped mean unless I also pass in an array for the counts (which is needlessly wasteful of memory). Stack array allocation (#851) would really help here, even the simple, 1d version.


Resolving patch:
diff --git a/numba/npyufunc/sigparse.py b/numba/npyufunc/sigparse.py
--- a/numba/npyufunc/sigparse.py
+++ b/numba/npyufunc/sigparse.py
@@ -46,7 +46,7 @@ def parse(src):
             else:
                 raise ValueError('bad token in signature "%s"' % tok[1])
 
-    ins, outs = stripws(sig).split('->')
+    ins, _, outs = stripws(sig).partition('->')
     inputs = list(parse(ins))
     outputs = list(parse(outs))
 
@@ -60,6 +60,6 @@ def parse(src):
 
     diff = osym.difference(isym)
     if diff:
-        raise NameError('undefined output symbols: %s' % ','.join(diff))
+        raise NameError('undefined output symbols: %s' % ','.join(sorted(diff)))
 
     return inputs, outputs


Test patch:
diff --git a/numba/tests/test_ufuncbuilding.py b/numba/tests/test_ufuncbuilding.py
--- a/numba/tests/test_ufuncbuilding.py
+++ b/numba/tests/test_ufuncbuilding.py
@@ -224,6 +224,14 @@ def test_guvectorize(self):
         self.assertTrue(numpy.all(a + a == b))
         self.assertEqual(b.dtype, numpy.dtype('int32'))
 
+    def test_guvectorize_no_output(self):
+        ufunc = guvectorize(['(int32[:,:], int32[:,:], int32[:,:])'],
+                            "(x,y),(x,y),(x,y)")(guadd)
+        a = numpy.arange(10, dtype='int32').reshape(2, 5)
+        out = numpy.zeros_like(a)
+        ufunc(a, a, out)
+        self.assertTrue(numpy.all(a + a == out))
+
     def test_guvectorize_objectmode(self):
         ufunc = guvectorize(['(int32[:,:], int32[:,:], int32[:,:])'],
                             "(x,y),(x,y)->(x,y)")(guadd_obj)
@@ -253,6 +261,22 @@ def test_guvectorize_identity(self):
         with self.assertRaises(ValueError):
             guvectorize(*args, identity=2)(add)
 
+    def test_guvectorize_invalid_layout(self):
+        sigs = ['(int32[:,:], int32[:,:], int32[:,:])']
+        # Syntax error
+        with self.assertRaises(ValueError) as raises:
+            guvectorize(sigs, ")-:")(guadd)
+        self.assertIn("bad token in signature", str(raises.exception))
+        # Output shape can't be inferred from inputs
+        with self.assertRaises(NameError) as raises:
+            guvectorize(sigs, "(x,y),(x,y)->(x,z,v)")(guadd)
+        self.assertEqual(str(raises.exception),
+                         "undefined output symbols: v,z")
+        # Arrow but no outputs
+        with self.assertRaises(ValueError) as raises:
+            guvectorize(sigs, "(x,y),(x,y),(x,y)->")(guadd)
+        # (error message depends on Numpy version)
+
 
 if __name__ == '__main__':
     unittest.main()