Repository: numba/numba

Issue:
Cannot use cffi.from_buffer when explicitly passing signature to JIT
I was experimenting with passing arrays to CFFI functions (see https://github.com/numba/numba/pull/1464) but found that it only works when not specifying the signature. This change makes the test fail in 0.23.1:

``` diff
diff --git a/numba/tests/test_cffi.py b/numba/tests/test_cffi.py
index ca304f9..fa1752d 100644
--- a/numba/tests/test_cffi.py
+++ b/numba/tests/test_cffi.py
@@ -82,7 +82,9 @@ class TestCFFI(TestCase):
         np.testing.assert_equal(pyfunc(x), cfunc(x))

     def test_pass_numpy_array_float32(self):
-        self._test_pass_numpy_array(vector_sin_float32, np.float32)
+        x = np.arange(10).astype(np.float32)
+        cfunc = jit('float32[:](float32[:])', nopython=True)(vector_sin_float32)
+        np.testing.assert_equal(vector_sin_float32(x), cfunc(x))

     def test_pass_numpy_array_float64(self):
         self._test_pass_numpy_array(vector_sin_float64, np.float64)
```

I was not able to try it with latest numba since it requires llvmlite 0.9.0 and it's not available in conda yet.

The workaround for me has been to provide a separate function without signature, but it's a pity because it is preventing me to provide different specializations for CFFI functions (see https://github.com/Pybonacci/cffi_test/).



Discussion:
By the way, I forgot to paste the error message:

``` bash
$ py.test test_cffi.py 
============================= test session starts ==============================
platform linux -- Python 3.5.1, pytest-2.8.5, py-1.4.31, pluggy-0.3.1
rootdir: /home/juanlu/Development/Python/numba, inifile: 
collected 11 items 

test_cffi.py .F.........

=================================== FAILURES ===================================
____________________ TestCFFI.test_pass_numpy_array_float32 ____________________

self = <numba.tests.test_cffi.TestCFFI testMethod=test_pass_numpy_array_float32>

    def test_pass_numpy_array_float32(self):
        x = np.arange(10).astype(np.float32)
>       cfunc = jit('float32[:](float32[:])', nopython=True)(vector_sin_float32)

test_cffi.py:86: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../decorators.py:168: in wrapper
    disp.compile(sig)
../dispatcher.py:349: in compile
    flags=flags, locals=self.locals)
../compiler.py:665: in compile_extra
    return pipeline.compile_extra(func)
../compiler.py:366: in compile_extra
    return self.compile_bytecode(bc, func_attr=self.func_attr)
../compiler.py:375: in compile_bytecode
    return self._compile_bytecode()
../compiler.py:652: in _compile_bytecode
    return pm.run(self.status)
../compiler.py:251: in run
    raise patched_exception
../compiler.py:243: in run
    res = stage()
../compiler.py:463: in stage_nopython_frontend
    self.locals)
../compiler.py:780: in type_inference_stage
    infer.propagate()
../typeinfer.py:565: in propagate
    raise errors[0]
../typeinfer.py:111: in propagate
    constraint(typeinfer)
../typeinfer.py:284: in __call__
    self.resolve(typeinfer, typevars, fnty)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <numba.typeinfer.CallConstraint object at 0x7f57e4b93dd8>
typeinfer = <numba.typeinfer.TypeInferer object at 0x7f57e4b09ef0>
typevars = {'$0.14': $0.14 := BoundFunction(ffi.from_buffer for ffi), '$0.10': $0.10 := BoundFunction(ffi.from_buffer for ffi), '....2 := Function(<built-in function empty_like>), '$0.4': $0.4 := array(float32, 1d, C), 'y': y := array(float32, 1d, C)}
fnty = BoundFunction(ffi.from_buffer for ffi)

    def resolve(self, typeinfer, typevars, fnty):
        assert fnty
        context = typeinfer.context

        n_pos_args = len(self.args)
        kwds = [kw for (kw, var) in self.kws]
        argtypes = [typevars[a.name] for a in self.args]
        argtypes += [typevars[var.name] for (kw, var) in self.kws]
        if self.vararg is not None:
            argtypes.append(typevars[self.vararg.name])

        if not all(a.defined for a in argtypes):
            # Cannot resolve call type until all argument types are known
            return

        args = tuple(a.getone() for a in argtypes)
        pos_args = args[:n_pos_args]
        if self.vararg is not None:
            if not isinstance(args[-1], types.BaseTuple):
                # Unsuitable for *args
                # (Python is more lenient and accepts all iterables)
                return
            pos_args += args[-1].types
            args = args[:-1]
        kw_args = dict(zip(kwds, args[n_pos_args:]))
        sig = context.resolve_function_type(fnty, pos_args, kw_args)
        if sig is None:
            desc = context.explain_function_type(fnty)
            headtemp = "Invalid usage of {0} with parameters ({1})"
            head = headtemp.format(fnty, ', '.join(map(str, args)))
            msg = '\n'.join([head, desc])
>           raise TypingError(msg, loc=self.loc)
E           numba.errors.TypingError: Failed at nopython (nopython frontend)
E           Invalid usage of BoundFunction(ffi.from_buffer for ffi) with parameters (array(float32, 1d, A))
E            * parameterized
E           File "cffi_usecases.py", line 130

../typeinfer.py:317: TypingError
===================== 1 failed, 10 passed in 1.50 seconds ======================

```

`float32[:]` designates a generic 1D array (presumed to be non-contiguous). A C-contiguous 1D array is `float32[::1]`. But in general it is recommended not to pass any explicit signature to `@jit`. Just let Numba infer the most adequate types.

In this case I needed to provide an explicit signature to generate different behaviours depending on types. If there is another way to achieve this I failed to see it in the documentation.

I cannot try your suggestion right now, but if it makes the code work would you accept a pull request with some clarifications in the docs?

What we could do is raise a clearer error when from_buffer() is used on a non-contiguous array.

We should also add some more discussion in the docs about how array types work, particularly the distinction between 'A', 'C' and 'F' layouts.


Resolving patch:
diff --git a/numba/targets/cffiimpl.py b/numba/targets/cffiimpl.py
--- a/numba/targets/cffiimpl.py
+++ b/numba/targets/cffiimpl.py
@@ -10,7 +10,7 @@
 
 registry = Registry()
 
-@registry.lower('ffi.from_buffer', types.Array)
+@registry.lower('ffi.from_buffer', types.Buffer)
 def from_buffer(context, builder, sig, args):
     assert len(sig.args) == 1
     assert len(args) == 1
diff --git a/numba/typing/cffi_utils.py b/numba/typing/cffi_utils.py
--- a/numba/typing/cffi_utils.py
+++ b/numba/typing/cffi_utils.py
@@ -9,6 +9,7 @@
 import ctypes
 
 from numba import types
+from numba.errors import TypingError
 from . import templates
 
 try:
@@ -63,7 +64,7 @@ def _type_map():
     global _cached_type_map
     if _cached_type_map is None:
         _cached_type_map = {
-            ffi.typeof('char') :                types.int8,
+            ffi.typeof('char') :                types.char,
             ffi.typeof('short') :               types.short,
             ffi.typeof('int') :                 types.intc,
             ffi.typeof('long') :                types.long_,
@@ -83,11 +84,6 @@ def _type_map():
             ffi.typeof('uint64_t') :            types.ulonglong,
             ffi.typeof('float') :               types.float_,
             ffi.typeof('double') :              types.double,
-            ffi.typeof('char *') :              types.voidptr,
-            ffi.typeof('void *') :              types.voidptr,
-            ffi.typeof('uint8_t *') :           types.CPointer(types.uint8),
-            ffi.typeof('float *') :             types.CPointer(types.float32),
-            ffi.typeof('double *') :            types.CPointer(types.float64),
             ffi.typeof('ssize_t') :             types.intp,
             ffi.typeof('size_t') :              types.uintp,
             ffi.typeof('void') :                types.void,
@@ -101,20 +97,24 @@ def map_type(cffi_type):
     """
     kind = getattr(cffi_type, 'kind', '')
     if kind in ('struct', 'union'):
-        raise TypeError("No support for struct or union")
+        raise TypeError("No support for CFFI %s" % (kind,))
     elif kind == 'function':
         if cffi_type.ellipsis:
             raise TypeError("vararg function is not supported")
         restype = map_type(cffi_type.result)
         argtypes = [map_type(arg) for arg in cffi_type.args]
         return templates.signature(restype, *argtypes)
+    elif kind == 'pointer':
+        pointee = cffi_type.item
+        if pointee.kind == 'void':
+            return types.voidptr
+        else:
+            return types.CPointer(map_type(pointee))
     else:
         result = _type_map().get(cffi_type)
-
-    if result is None:
-        raise TypeError(cffi_type)
-
-    return result
+        if result is None:
+            raise TypeError(cffi_type)
+        return result
 
 
 def make_function_type(cffi_func):
@@ -147,11 +147,15 @@ class FFI_from_buffer(templates.AbstractTemplate):
     key = 'ffi.from_buffer'
 
     def generic(self, args, kws):
-        if kws or (len(args) != 1):
+        if kws or len(args) != 1:
             return
         [ary] = args
-        if not (isinstance(ary, types.Array) and ary.layout in ('C', 'F')):
-            return
+        if not isinstance(ary, types.Buffer):
+            raise TypingError("from_buffer() expected a buffer object, got %s"
+                              % (ary,))
+        if ary.layout not in ('C', 'F'):
+            raise TypingError("from_buffer() unsupported on non-contiguous buffers (got %s)"
+                              % (ary,))
         ptr = types.CPointer(ary.dtype)
         return templates.signature(ptr, ary)
 


Test patch:
diff --git a/numba/tests/cffi_usecases.py b/numba/tests/cffi_usecases.py
--- a/numba/tests/cffi_usecases.py
+++ b/numba/tests/cffi_usecases.py
@@ -141,12 +141,8 @@ def use_user_defined_symbols():
 # (cffi_usecases_ool.ffi is a CompiledFFI object) so we use both in these
 # functions.
 
-def vector_sin_float32(x):
-    y = np.empty_like(x)
+def vector_sin_float32(x, y):
     vsSin(len(x), ffi.from_buffer(x), ffi_ool.from_buffer(y))
-    return y
 
-def vector_sin_float64(x):
-    y = np.empty_like(x)
+def vector_sin_float64(x, y):
     vdSin(len(x), ffi.from_buffer(x), ffi_ool.from_buffer(y))
-    return y
diff --git a/numba/tests/test_cffi.py b/numba/tests/test_cffi.py
--- a/numba/tests/test_cffi.py
+++ b/numba/tests/test_cffi.py
@@ -1,9 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
+import array
 import numpy as np
+import sys
 
 from numba import unittest_support as unittest
-from numba import jit, cffi_support, types
+from numba import jit, cffi_support, types, errors
 from numba.compiler import compile_isolated, Flags
 from numba.tests.support import TestCase
 
@@ -86,16 +88,41 @@ def test_user_defined_symbols(self):
         cfunc = jit(nopython=True)(pyfunc)
         self.assertEqual(pyfunc(), cfunc())
 
-    def _test_pass_numpy_array(self, pyfunc, dtype):
+    def check_vector_sin(self, cfunc, x, y):
+        cfunc(x, y)
+        np.testing.assert_allclose(y, np.sin(x))
+
+    def _test_from_buffer_numpy_array(self, pyfunc, dtype):
         x = np.arange(10).astype(dtype)
+        y = np.zeros_like(x)
         cfunc = jit(nopython=True)(pyfunc)
-        np.testing.assert_equal(pyfunc(x), cfunc(x))
+        self.check_vector_sin(cfunc, x, y)
+
+    def test_from_buffer_float32(self):
+        self._test_from_buffer_numpy_array(mod.vector_sin_float32, np.float32)
 
-    def test_pass_numpy_array_float32(self):
-        self._test_pass_numpy_array(mod.vector_sin_float32, np.float32)
+    def test_from_buffer_float64(self):
+        self._test_from_buffer_numpy_array(mod.vector_sin_float64, np.float64)
 
-    def test_pass_numpy_array_float64(self):
-        self._test_pass_numpy_array(mod.vector_sin_float64, np.float64)
+    @unittest.skipIf(sys.version_info < (3,),
+                     "buffer protocol on array.array needs Python 3+")
+    def test_from_buffer_pyarray(self):
+        pyfunc = mod.vector_sin_float32
+        cfunc = jit(nopython=True)(pyfunc)
+        x = array.array("f", range(10))
+        y = array.array("f", [0] * len(x))
+        self.check_vector_sin(cfunc, x, y)
+
+    def test_from_buffer_error(self):
+        pyfunc = mod.vector_sin_float32
+        cfunc = jit(nopython=True)(pyfunc)
+        # Non-contiguous array
+        x = np.arange(10).astype(np.float32)[::2]
+        y = np.zeros_like(x)
+        with self.assertRaises(errors.TypingError) as raises:
+            cfunc(x, y)
+        self.assertIn("from_buffer() unsupported on non-contiguous buffers",
+                      str(raises.exception))
 
 
 if __name__ == '__main__':