Repository: cupy/cupy

Issue:
Support peer access
```py
import cupy as cp


with cp.cuda.Device(0):
    d0 = cp.arange(10)
with cp.cuda.Device(1):
    # Enable dereferencing pointer of device 0 in the current device.
    # otherwise you will get `cudaErrorIllegalAddress: an illegal memory access was encountered`
    cp.cuda.runtime.deviceEnablePeerAccess(0)
    d1 = d0 * 2

print(d1)
```

Currently, this code fails with the error `Array device must be same as the current device: array device = 0 while current = 1`, but it should work. Maybe changing the error to the warning?

ref: https://stackoverflow.com/questions/68074985/read-write-a-cupy-array-on-multiple-gpus

Note: `deviceEnablePeerAccess` may not work with all combinations of GPUs.


Discussion:
Another idea would be to enable peer access automatically as we do when doing D2D memcpy.
https://github.com/cupy/cupy/blob/v10.0.0a1/cupy/cuda/memory.pyx#L604-L624

Resolving patch:
diff --git a/examples/peer/peer_matrix.py b/examples/peer/peer_matrix.py
new file mode 100644
--- /dev/null
+++ b/examples/peer/peer_matrix.py
@@ -0,0 +1,17 @@
+import cupy
+
+
+def main():
+    gpus = cupy.cuda.runtime.getDeviceCount()
+    for peerDevice in range(gpus):
+        for device in range(gpus):
+            if peerDevice == device:
+                continue
+            flag = cupy.cuda.runtime.deviceCanAccessPeer(device, peerDevice)
+            print(
+                f'Can access #{peerDevice} memory from #{device}: '
+                f'{flag == 1}')
+
+
+if __name__ == '__main__':
+    main()


Test patch:
diff --git a/tests/cupy_tests/core_tests/test_elementwise.py b/tests/cupy_tests/core_tests/test_elementwise.py
--- a/tests/cupy_tests/core_tests/test_elementwise.py
+++ b/tests/cupy_tests/core_tests/test_elementwise.py
@@ -1,6 +1,7 @@
 import unittest
 
 import numpy
+import pytest
 
 import cupy
 from cupy import _core
@@ -26,10 +27,20 @@ def test_copy(self, dtype):
 
     @testing.multi_gpu(2)
     @testing.for_all_dtypes()
-    def test_copy_multigpu(self, dtype):
+    def test_copy_multigpu_nopeer(self, dtype):
+        if cuda.runtime.deviceCanAccessPeer(0, 1) == 1:
+            pytest.skip('peer access is available')
         with self.assertRaises(ValueError):
             self.check_copy(dtype, 0, 1)
 
+    @testing.multi_gpu(2)
+    @testing.for_all_dtypes()
+    def test_copy_multigpu_peer(self, dtype):
+        if cuda.runtime.deviceCanAccessPeer(0, 1) != 1:
+            pytest.skip('peer access is unavailable')
+        with pytest.warns(cupy._util.PerformanceWarning):
+            self.check_copy(dtype, 0, 1)
+
     @testing.for_orders('CFAK')
     @testing.for_all_dtypes()
     @testing.numpy_cupy_allclose()
diff --git a/tests/cupy_tests/manipulation_tests/test_basic.py b/tests/cupy_tests/manipulation_tests/test_basic.py
--- a/tests/cupy_tests/manipulation_tests/test_basic.py
+++ b/tests/cupy_tests/manipulation_tests/test_basic.py
@@ -75,17 +75,32 @@ def get_numpy():
             with cuda.Device(dev3):
                 c = testing.shaped_arange((2, 3, 4), cupy, '?')
             with cuda.Device(dev4):
-                with pytest.raises(
-                        ValueError,
-                        match='^Array device must be same as the '
-                        'current device'):
-                    cupy.copyto(a, b, where=c)
+                if all([(peer == dev4) or
+                        (cuda.runtime.deviceCanAccessPeer(dev4, peer) == 1)
+                        for peer in (dev1, dev2, dev3)]):
+                    with pytest.warns(cupy._util.PerformanceWarning):
+                        cupy.copyto(a, b, where=c)
+                else:
+                    with pytest.raises(
+                            ValueError,
+                            match='Peer access is unavailable'):
+                        cupy.copyto(a, b, where=c)
 
     @testing.multi_gpu(2)
     @testing.for_all_dtypes()
     def test_copyto_where_multigpu_raises(self, dtype):
         self._check_copyto_where_multigpu_raises(dtype, 2)
 
+    @testing.multi_gpu(4)
+    @testing.for_all_dtypes()
+    def test_copyto_where_multigpu_raises_4(self, dtype):
+        self._check_copyto_where_multigpu_raises(dtype, 4)
+
+    @testing.multi_gpu(6)
+    @testing.for_all_dtypes()
+    def test_copyto_where_multigpu_raises_6(self, dtype):
+        self._check_copyto_where_multigpu_raises(dtype, 6)
+
     @testing.multi_gpu(2)
     @testing.for_all_dtypes()
     @testing.numpy_cupy_array_equal()
diff --git a/tests/cupy_tests/manipulation_tests/test_join.py b/tests/cupy_tests/manipulation_tests/test_join.py
--- a/tests/cupy_tests/manipulation_tests/test_join.py
+++ b/tests/cupy_tests/manipulation_tests/test_join.py
@@ -96,8 +96,12 @@ def test_concatenate_large_different_devices(self):
         for i in range(10):
             with cuda.Device(i % 2):
                 arrs.append(cupy.empty((2, 3, 4)))
-        with pytest.raises(ValueError):
-            cupy.concatenate(arrs)
+        if cuda.runtime.deviceCanAccessPeer(0, 1) == 1:
+            with pytest.warns(cupy._util.PerformanceWarning):
+                cupy.concatenate(arrs)
+        else:
+            with pytest.raises(ValueError):
+                cupy.concatenate(arrs)
 
     @testing.for_all_dtypes(name='dtype')
     @testing.numpy_cupy_array_equal()