Repository: numba/numba

Issue:
Storing ctypes c_void_p type result in numpy array: Cannot cast void* to uint64
I am trying to use numba to call a ctypes function in a jitted nopython function. The following problem arises when the ctypes function return type is `c_void_p` (I did not have problems with other ctypes functions that return a float or a byte).

Without `nopython=True` it works fine:

```python
## Create some random polygon data

from shapely.geometry import Point, Polygon
import random

polygons = np.array([
    Polygon([(i + random.random(), j + random.random()) for _ in range(3)])
    for (i, j) in [(random.random()*10, random.random()*10) for _ in range(1000)]
], dtype=object)

# create uint64 array of pointers to the C objects
polygons_pointers = np.array([p._geom for p in polygons], dtype=np.uintp)
```

```python
import numba
from shapely.geos import lgeos
geos_handle = lgeos.geos_handle

# ctypes wrapped c function to call
GEOSIntersection = lgeos._lgeos.GEOSIntersection_r

## the actual function: calling the ctypes GEOSIntersection function in a loop
## and store the results in a new array

@numba.jit
def intersection(geoms1, geom2):

    n = geoms1.shape[0]
    res = np.empty(n, dtype=np.uintp)
    
    for i in range(n):
        res[i] = GEOSIntersection(geos_handle, int(geoms1[i]), int(geom2))
        
    return res

# calling the function on the random data
res = intersection(polygons_pointers, polygons_pointers[0])
```

So the above works fine (in any case without error), but when adding `nopython=True` to the jit call, I get an error:

```
...
NotImplementedError: (<numba.targets.base.OverloadSelector object at 0x7fae4f464240>, (void*, uint64))
...

NotImplementedError: Cannot cast void* to uint64: %".360" = load i8*, i8** %"$53.12"
...
LoweringError: Failed at nopython (nopython mode backend)
Cannot cast void* to uint64: %".360" = load i8*, i8** %"$53.12"
```

(full traceback below in the details) I tried to use other dtypes for `res`, but without luck.

The ctypes wrapped function accepts / returns 

```python
>>> GEOSIntersection.argtypes
[ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
>>> GEOSIntersection.restype
ctypes.c_void_p
```

So for the input passing an integer for a `c_void_p` argument is fine, but for the output then storing such a `c_void_p` result in a integer array does not work in `nopython` mode. 

I tried to make the reproducible example as simple as possible, but the above still requires shapely (I suppose I could write a small custom C function and wrap it in ctypes?).

I am using the latest (of two weeks ago) numba:

```python
>>> numba.__version__
'0.40.0dev0+58.g1a1a738'
```

<details>

```
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/base.py in cast(self, builder, val, fromty, toty)
    674         try:
--> 675             impl = self._casts.find((fromty, toty))
    676             return impl(self, builder, fromty, toty, val)

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/base.py in find(self, sig)
     47         if out is None:
---> 48             out = self._find(sig)
     49             self._cache[sig] = out

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/base.py in _find(self, sig)
     56         else:
---> 57             raise NotImplementedError(self, sig)
     58 

NotImplementedError: (<numba.targets.base.OverloadSelector object at 0x7fae4f464240>, (void*, uint64))

During handling of the above exception, another exception occurred:

NotImplementedError                       Traceback (most recent call last)
~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    594     try:
--> 595         yield
    596     except NumbaError as e:

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_block(self, block)
    253                                    loc=self.loc, errcls_=defaulterrcls):
--> 254                 self.lower_inst(inst)
    255 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_inst(self, inst)
    357             assert signature is not None
--> 358             return self.lower_setitem(inst.target, inst.index, inst.value, signature)
    359 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_setitem(self, target_var, index_var, value_var, signature)
    429 
--> 430         return impl(self.builder, (target, index, value))
    431 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/base.py in __call__(self, builder, args)
   1078     def __call__(self, builder, args):
-> 1079         return self._imp(self._context, builder, self._sig, args)
   1080 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/arrayobj.py in setitem_array(context, builder, sig, args)
    481     # Store source value the given location
--> 482     val = context.cast(builder, val, valty, aryty.dtype)
    483     store_item(context, builder, aryty, val, dataptr)

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/targets/base.py in cast(self, builder, val, fromty, toty)
    678             raise NotImplementedError(
--> 679                 "Cannot cast %s to %s: %s" % (fromty, toty, val))
    680 

NotImplementedError: Cannot cast void* to uint64: %".360" = load i8*, i8** %"$53.12"

During handling of the above exception, another exception occurred:

LoweringError                             Traceback (most recent call last)
<ipython-input-18-88e9a5619471> in <module>()
----> 1 res = intersection(polygons_pointers,polygons_pointers[0])

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    366                     e.patch_message(''.join(e.args) + help_msg)
    367             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 368             raise e
    369 
    370     def inspect_llvm(self, signature=None):

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    323                 argtypes.append(self.typeof_pyval(a))
    324         try:
--> 325             return self.compile(tuple(argtypes))
    326         except errors.TypingError as e:
    327             # Intercept typing error that may be due to an argument

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/dispatcher.py in compile(self, sig)
    651 
    652                 self._cache_misses[sig] += 1
--> 653                 cres = self._compiler.compile(args, return_type)
    654                 self.add_overload(cres)
    655                 self._cache.save_overload(sig, cres)

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     81                                       args=args, return_type=return_type,
     82                                       flags=flags, locals=self.locals,
---> 83                                       pipeline_class=self.pipeline_class)
     84         # Check typing error if object mode is used
     85         if cres.typing_error is not None and not flags.enable_pyobject:

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    879     pipeline = pipeline_class(typingctx, targetctx, library,
    880                               args, return_type, flags, locals)
--> 881     return pipeline.compile_extra(func)
    882 
    883 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in compile_extra(self, func)
    366         self.lifted = ()
    367         self.lifted_from = None
--> 368         return self._compile_bytecode()
    369 
    370     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in _compile_bytecode(self)
    810         """
    811         assert self.func_ir is None
--> 812         return self._compile_core()
    813 
    814     def _compile_ir(self):

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in _compile_core(self)
    797         self.define_pipelines(pm)
    798         pm.finalize()
--> 799         res = pm.run(self.status)
    800         if res is not None:
    801             # Early pipeline completion

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    252                     # No more fallback pipelines?
    253                     if is_final_pipeline:
--> 254                         raise patched_exception
    255                     # Go to next fallback pipeline
    256                     else:

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    244                 try:
    245                     event(stage_name)
--> 246                     stage()
    247                 except _EarlyPipelineCompletion as e:
    248                     return e.result

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in stage_nopython_backend(self)
    677         """
    678         lowerfn = self.backend_nopython_mode
--> 679         self._backend(lowerfn, objectmode=False)
    680 
    681     def stage_compile_interp_mode(self):

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in _backend(self, lowerfn, objectmode)
    627             self.library.enable_object_caching()
    628 
--> 629         lowered = lowerfn()
    630         signature = typing.signature(self.return_type, *self.args)
    631         self.cr = compile_result(

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in backend_nopython_mode(self)
    614                 self.return_type,
    615                 self.calltypes,
--> 616                 self.flags)
    617 
    618     def _backend(self, lowerfn, objectmode):

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/compiler.py in native_lowering_stage(targetctx, library, interp, typemap, restype, calltypes, flags)
    998 
    999     lower = lowering.Lower(targetctx, library, fndesc, interp)
-> 1000     lower.lower()
   1001     if not flags.no_cpython_wrapper:
   1002         lower.create_cpython_wrapper(flags.release_gil)

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower(self)
    171         if self.generator_info is None:
    172             self.genlower = None
--> 173             self.lower_normal_function(self.fndesc)
    174         else:
    175             self.genlower = self.GeneratorLower(self)

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_normal_function(self, fndesc)
    212         # Init argument values
    213         self.extract_function_arguments()
--> 214         entry_block_tail = self.lower_function_body()
    215 
    216         # Close tail of entry block

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_function_body(self)
    237             bb = self.blkmap[offset]
    238             self.builder.position_at_end(bb)
--> 239             self.lower_block(block)
    240 
    241         self.post_lower()

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/lowering.py in lower_block(self, block)
    252             with new_error_context('lowering "{inst}" at {loc}', inst=inst,
    253                                    loc=self.loc, errcls_=defaulterrcls):
--> 254                 self.lower_inst(inst)
    255 
    256     def create_cpython_wrapper(self, release_gil=False):

~/miniconda3/envs/dev/lib/python3.5/contextlib.py in __exit__(self, type, value, traceback)
     75                 value = type()
     76             try:
---> 77                 self.gen.throw(type, value, traceback)
     78             except StopIteration as exc:
     79                 # Suppress StopIteration *unless* it's the same exception that

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    601         from numba import config
    602         tb = sys.exc_info()[2] if config.FULL_TRACEBACKS else None
--> 603         six.reraise(type(newerr), newerr, tb)
    604 
    605 

~/miniconda3/envs/dev/lib/python3.5/site-packages/numba/six.py in reraise(tp, value, tb)
    657         if value.__traceback__ is not tb:
    658             raise value.with_traceback(tb)
--> 659         raise value
    660 
    661 else:

LoweringError: Failed at nopython (nopython mode backend)
Cannot cast void* to uint64: %".360" = load i8*, i8** %"$53.12"

File "<ipython-input-17-af18922cb926>", line 8:
def intersection(geoms1, geom2):
    <source elided>
    for i in range(n):
        res[i] = GEOSIntersection(geos_handle, int(geoms1[i]), int(geom2))
        ^
[1] During: lowering "res[i] = $53.12" at <ipython-input-17-af18922cb926> (8)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.

Please report the error message and traceback, along with a minimal reproducer
at: https://github.com/numba/numba/issues/new

If more help is needed please feel free to speak to the Numba core developers
directly at: https://gitter.im/numba/numba

Thanks in advance for your help in improving Numba!
```

</details>


Discussion:
Notebook with the reproducible example: http://nbviewer.ipython.org/fd32ded92c62b4fd67782f1ccbc36c78

Resolving patch:
diff --git a/numba/typing/ctypes_utils.py b/numba/typing/ctypes_utils.py
--- a/numba/typing/ctypes_utils.py
+++ b/numba/typing/ctypes_utils.py
@@ -14,7 +14,7 @@
 
 _FROM_CTYPES = {
     ctypes.c_bool: types.boolean,
-    
+
     ctypes.c_int8:  types.int8,
     ctypes.c_int16: types.int16,
     ctypes.c_int32: types.int32,
@@ -113,6 +113,9 @@ def make_function_type(cfnptr):
     cargs = [from_ctypes(a)
              for a in cfnptr.argtypes]
     cret = from_ctypes(cfnptr.restype)
+    # void* return type is implicitly converted to intp to match python
+    if cret == types.voidptr:
+        cret = types.intp
     if sys.platform == 'win32' and not cfnptr._flags_ & ctypes._FUNCFLAG_CDECL:
         # 'stdcall' calling convention under Windows
         cconv = 'x86_stdcallcc'


Test patch:
diff --git a/numba/tests/test_ctypes.py b/numba/tests/test_ctypes.py
--- a/numba/tests/test_ctypes.py
+++ b/numba/tests/test_ctypes.py
@@ -230,6 +230,37 @@ def test_passing_array_ctypes_voidptr_pass_ptr(self):
         self.assertIn("Invalid use of ExternalFunctionPointer",
                       str(raises.exception))
 
+    def test_storing_voidptr_to_int_array(self):
+        # Make C callback that returns a void*
+        cproto = CFUNCTYPE(c_void_p)
+
+        @cproto
+        def get_voidstar():
+            return 0xdeadbeef
+
+        # Make python functions that use the C callback
+        def pyfunc(a):
+            ptr = get_voidstar()
+            a[0] = ptr
+            return ptr
+
+        # Compile it
+        cres = compile_isolated(pyfunc, [types.intp[::1]])
+        cfunc = cres.entry_point
+
+        # Setup inputs
+        arr_got = np.zeros(1, dtype=np.intp)
+        arr_expect = arr_got.copy()
+
+        # Run functions
+        ret_got = cfunc(arr_got)
+        ret_expect = pyfunc(arr_expect)
+
+        # Check
+        self.assertEqual(ret_expect, 0xdeadbeef)
+        self.assertPreciseEqual(ret_got, ret_expect)
+        self.assertPreciseEqual(arr_got, arr_expect)
+
 
 if __name__ == '__main__':
     unittest.main()