Repository: numba/numba

Issue:
Exception using `.get_ipc_handle(...)` on array from `as_cuda_array(...)
## Reporting a bug

- [x] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

<hr>


Am trying to access `.get_ipc_handle(...)` on a Numba `DeviceNDArray` created through `as_cuda_array(...)`. Here's some code using CuPy that demonstrates this use case:

```python
import cupy
import numba.cuda

a = cupy.arange(5)
a2 = numba.cuda.as_cuda_array(a)

a2.get_ipc_handle()
```

However am getting the following exception when calling `.get_ipc_handle(...)`:

```python
Traceback (most recent call last):
  File "script.py", line 7, in <module>
    a2.get_ipc_handle()
  File "/datasets/jkirkham/miniconda/envs/rapids13dev/lib/python3.7/site-packages/numba/cuda/cudadrv/devicearray.py", line 298, in get_ipc_handle
    ipch = devices.get_context().get_ipc_handle(self.gpu_data)
  File "/datasets/jkirkham/miniconda/envs/rapids13dev/lib/python3.7/site-packages/numba/cuda/cudadrv/driver.py", line 847, in get_ipc_handle
    memory.owner.handle,
AttributeError: 'cupy.core.core.ndarray' object has no attribute 'handle'
```

It appears this code that is trying to compute the `offset` causes the issue

https://github.com/numba/numba/blob/cc7e7c7cfa6389b54d3b5c2c95751c97eb531a96/numba/cuda/cudadrv/driver.py#L850

cc @Iroy30 @kkraus14


Discussion:
@gmarkall any chance you could take a look at this please? Thanks.
Already looking into it :-)
> Already looking into it :-)

Brilliant, thanks!
The use of the owner of a `MemoryPointer` is a bit overloaded:

1. It's used to keep the owning object alive by holding a reference to it - as is set up by `as_cuda_array`.
2. It's also expected to be a `MemoryPointer` which points to the base of the allocation - this made sense when Numba was "isolated" and didn't communicate with other libraries through the CUDA array interface, but doesn't make as much sense now.

I think the second use can be replaced with a call to `cuMemGetAddressRange` instead, to disentangle these two uses, and also allow getting the base of the allocation for pointers that come in through the CUDA array interface - I'm experimenting with a fix now.

Resolving patch:
diff --git a/numba/cuda/cudadrv/driver.py b/numba/cuda/cudadrv/driver.py
--- a/numba/cuda/cudadrv/driver.py
+++ b/numba/cuda/cudadrv/driver.py
@@ -834,13 +834,11 @@ def get_memory_info(self):
         return MemoryInfo(free=free.value, total=total.value)
 
     def get_ipc_handle(self, memory):
+        base, end = device_extents(memory)
         ipchandle = drvapi.cu_ipc_mem_handle()
-        driver.cuIpcGetMemHandle(
-            byref(ipchandle),
-            memory.owner.handle,
-        )
+        driver.cuIpcGetMemHandle(byref(ipchandle), base)
         source_info = self.context.device.get_device_identity()
-        offset = memory.handle.value - memory.owner.handle.value
+        offset = memory.handle.value - base
 
         return IpcHandle(memory, ipchandle, memory.size, source_info,
                          offset=offset)
@@ -2075,7 +2073,7 @@ def get_devptr_for_active_ctx(ptr):
     """Query the device pointer usable in the current context from an arbitrary
     pointer.
     """
-    devptr = c_void_p(0)
+    devptr = drvapi.cu_device_ptr()
     if ptr != 0:
         attr = enums.CU_POINTER_ATTRIBUTE_DEVICE_POINTER
         driver.cuPointerGetAttribute(byref(devptr), attr, ptr)


Test patch:
diff --git a/numba/cuda/testing.py b/numba/cuda/testing.py
--- a/numba/cuda/testing.py
+++ b/numba/cuda/testing.py
@@ -91,3 +91,15 @@ def captured_cuda_stdout():
         with redirect_c_stdout() as stream:
             yield CUDATextCapture(stream)
             cuda.synchronize()
+
+
+class ForeignArray(object):
+    """
+    Class for emulating an array coming from another library through the CUDA
+    Array interface. This just hides a DeviceNDArray so that it doesn't look
+    like a DeviceNDArray.
+    """
+
+    def __init__(self, arr):
+        self._arr = arr
+        self.__cuda_array_interface__ = arr.__cuda_array_interface__
diff --git a/numba/cuda/tests/cudapy/test_cuda_array_interface.py b/numba/cuda/tests/cudapy/test_cuda_array_interface.py
--- a/numba/cuda/tests/cudapy/test_cuda_array_interface.py
+++ b/numba/cuda/tests/cudapy/test_cuda_array_interface.py
@@ -2,16 +2,10 @@
 
 from numba import vectorize, guvectorize
 from numba import cuda
-from numba.cuda.testing import unittest, ContextResettingTestCase
+from numba.cuda.testing import unittest, ContextResettingTestCase, ForeignArray
 from numba.cuda.testing import skip_on_cudasim, skip_if_external_memmgr
 
 
-class MyArray(object):
-    def __init__(self, arr):
-        self._arr = arr
-        self.__cuda_array_interface__ = arr.__cuda_array_interface__
-
-
 @skip_on_cudasim('CUDA Array Interface is not supported in the simulator')
 class TestCudaArrayInterface(ContextResettingTestCase):
     def test_as_cuda_array(self):
@@ -19,7 +13,7 @@ def test_as_cuda_array(self):
         self.assertFalse(cuda.is_cuda_array(h_arr))
         d_arr = cuda.to_device(h_arr)
         self.assertTrue(cuda.is_cuda_array(d_arr))
-        my_arr = MyArray(d_arr)
+        my_arr = ForeignArray(d_arr)
         self.assertTrue(cuda.is_cuda_array(my_arr))
         wrapped = cuda.as_cuda_array(my_arr)
         self.assertTrue(cuda.is_cuda_array(wrapped))
@@ -58,7 +52,7 @@ def test_ownership(self):
     def test_kernel_arg(self):
         h_arr = np.arange(10)
         d_arr = cuda.to_device(h_arr)
-        my_arr = MyArray(d_arr)
+        my_arr = ForeignArray(d_arr)
         wrapped = cuda.as_cuda_array(my_arr)
 
         @cuda.jit
@@ -78,13 +72,13 @@ def vadd(a, b):
 
         # Case 1: use custom array as argument
         h_arr = np.random.random(10)
-        arr = MyArray(cuda.to_device(h_arr))
+        arr = ForeignArray(cuda.to_device(h_arr))
         val = 6
         out = vadd(arr, val)
         np.testing.assert_array_equal(out.copy_to_host(), h_arr + val)
 
         # Case 2: use custom array as return
-        out = MyArray(cuda.device_array(h_arr.shape))
+        out = ForeignArray(cuda.device_array(h_arr.shape))
         returned = vadd(h_arr, val, out=out)
         np.testing.assert_array_equal(returned.copy_to_host(), h_arr + val)
 
@@ -95,13 +89,13 @@ def vadd(inp, val, out):
 
         # Case 1: use custom array as argument
         h_arr = np.random.random(10)
-        arr = MyArray(cuda.to_device(h_arr))
+        arr = ForeignArray(cuda.to_device(h_arr))
         val = np.float64(7)
         out = vadd(arr, val)
         np.testing.assert_array_equal(out.copy_to_host(), h_arr + val)
 
         # Case 2: use custom array as return
-        out = MyArray(cuda.device_array(h_arr.shape))
+        out = ForeignArray(cuda.device_array(h_arr.shape))
         returned = vadd(h_arr, val, out=out)
         np.testing.assert_array_equal(returned.copy_to_host(), h_arr + val)
         self.assertEqual(returned.device_ctypes_pointer.value,
@@ -233,7 +227,7 @@ def add_one(arr):
             if x < N:
                 arr[x] += 1
 
-        d_arr = MyArray(c_arr)
+        d_arr = ForeignArray(c_arr)
         add_one[1, 10](d_arr)  # this should pass
 
     def test_strides(self):
diff --git a/numba/cuda/tests/cudapy/test_ipc.py b/numba/cuda/tests/cudapy/test_ipc.py
--- a/numba/cuda/tests/cudapy/test_ipc.py
+++ b/numba/cuda/tests/cudapy/test_ipc.py
@@ -1,5 +1,6 @@
 import sys
 import multiprocessing as mp
+import itertools
 import traceback
 import pickle
 
@@ -7,7 +8,8 @@
 
 from numba import cuda
 from numba.cuda.cudadrv import drvapi, devicearray
-from numba.cuda.testing import skip_on_cudasim, ContextResettingTestCase
+from numba.cuda.testing import (skip_on_cudasim, ContextResettingTestCase,
+                                ForeignArray)
 from numba.tests.support import linux_only
 import unittest
 
@@ -109,12 +111,22 @@ def test_ipc_handle(self):
             np.testing.assert_equal(arr, out)
         proc.join(3)
 
-    def check_ipc_handle_serialization(self, index_arg=None):
+    def variants(self):
+        # Test with no slicing and various different slices
+        indices = (None, slice(3, None), slice(3, 8), slice(None, 8))
+        # Test with a Numba DeviceNDArray, or an array from elsewhere through
+        # the CUDA Array Interface
+        foreigns = (False, True)
+        return itertools.product(indices, foreigns)
+
+    def check_ipc_handle_serialization(self, index_arg=None, foreign=False):
         # prepare data for IPC
         arr = np.arange(10, dtype=np.intp)
         devarr = cuda.to_device(arr)
         if index_arg is not None:
             devarr = devarr[index_arg]
+        if foreign:
+            devarr = cuda.as_cuda_array(ForeignArray(devarr))
         expect = devarr.copy_to_host()
 
         # create IPC handle
@@ -142,20 +154,19 @@ def check_ipc_handle_serialization(self, index_arg=None):
         proc.join(3)
 
     def test_ipc_handle_serialization(self):
-        # test no slicing
-        self.check_ipc_handle_serialization()
-        # slicing tests
-        self.check_ipc_handle_serialization(slice(3, None))
-        self.check_ipc_handle_serialization(slice(3, 8))
-        self.check_ipc_handle_serialization(slice(None, 8))
-
-    def check_ipc_array(self, index_arg=None):
+        for index, foreign, in self.variants():
+            with self.subTest(index=index, foreign=foreign):
+                self.check_ipc_handle_serialization(index, foreign)
+
+    def check_ipc_array(self, index_arg=None, foreign=False):
         # prepare data for IPC
         arr = np.arange(10, dtype=np.intp)
         devarr = cuda.to_device(arr)
         # Slice
         if index_arg is not None:
             devarr = devarr[index_arg]
+        if foreign:
+            devarr = cuda.as_cuda_array(ForeignArray(devarr))
         expect = devarr.copy_to_host()
         ipch = devarr.get_ipc_handle()
 
@@ -173,12 +184,10 @@ def check_ipc_array(self, index_arg=None):
         proc.join(3)
 
     def test_ipc_array(self):
-        # test no slicing
-        self.check_ipc_array()
-        # slicing tests
-        self.check_ipc_array(slice(3, None))
-        self.check_ipc_array(slice(3, 8))
-        self.check_ipc_array(slice(None, 8))
+        for index, foreign, in self.variants():
+            with self.subTest(index=index, foreign=foreign):
+                self.check_ipc_array(index, foreign)
+
 
 @unittest.skipIf(linux, 'Only on OS other than Linux')
 @skip_on_cudasim('Ipc not available in CUDASIM')
@@ -196,8 +205,6 @@ def staged_ipc_handle_test(handle, device_num, result_queue):
     def the_work():
         with cuda.gpus[device_num]:
             this_ctx = cuda.devices.get_context()
-            can_access = handle.can_access_peer(this_ctx)
-            print('can_access_peer {} {}'.format(this_ctx, can_access))
             deviceptr = handle.open_staged(this_ctx)
             arrsize = handle.size // np.dtype(np.intp).itemsize
             hostarray = np.zeros(arrsize, dtype=np.intp)
@@ -214,7 +221,6 @@ def staged_ipc_array_test(ipcarr, device_num, result_queue):
     try:
         with cuda.gpus[device_num]:
             this_ctx = cuda.devices.get_context()
-            print(this_ctx.device)
             with ipcarr as darr:
                 arr = darr.copy_to_host()
                 try: