Repository: numba/numba

Issue:
UNKNOWN_CUDA_ERROR depends on magic print statement in CUDASIM
Unfortunately I found a strange bug on Numba 0.30.1. Somehow commenting out the print statement on line 47 results in UNKNOWN_CUDA_ERROR even if it is preceded and followed by the same statement, but it runs perfectly when including the print. A similar crash occurs outside of CUDASIM too.

    import numpy as np
    
    features = np.ones((20736,2), dtype=np.float64)
    targets = np.ones(20736, dtype=np.float64) + np.e + 1
    
    ###Numba###
    num_examples, num_weights = features.shape
    
    elements_per_example = num_weights + 1
    bytes_per_element = 8
    
    
    num_SMPs = 12
    num_blocks = num_SMPs * 4
    max_threads = min(2048 * num_SMPs, 1024 * num_blocks)
    
    shared_memory_per_SMP = 48*1024
    shared_memory_per_block = int( min(shared_memory_per_SMP, shared_memory_per_SMP * num_SMPs / num_blocks) * 0.01 ) # Use 1% of available memory
    
    weights_memory = num_weights * bytes_per_element
    
    max_elements_per_block = int( (shared_memory_per_block-weights_memory) // bytes_per_element )
    
    examples_per_block = int( max_elements_per_block // elements_per_example )
    
    threads_per_block = examples_per_block
    
    print("num_weights:", num_weights)
    print("examples_per_block:", examples_per_block)
    print("threads_per_block:", threads_per_block)
    print("num_blocks:", num_blocks)
    ###Numba###
    
    from numba import cuda, float64, vectorize
    from time import time
    
    minimum_delta = 1e-10
    MAX_EXPONENT = int(round(np.log(1000/minimum_delta) / np.log(3)))
    
    @cuda.jit(device=True)
    def getCost(s_features, s_weights, s_targets, s_costs, s_initialcost):
    
        threadIdx = cuda.threadIdx.x
        prediction = 0
        print("prediction:", prediction)
        for i in range(num_weights):
            print("prediction:", prediction)  ######### Magic print statement ###########
            prediction += s_features[threadIdx,i] * s_weights[i]
        print("prediction:", prediction)
        cost = abs(prediction - s_targets[threadIdx])
        s_initialcost[0] = 777
        print("threadIdx:", threadIdx)
    
    class descend:
        def __new__(this, features, targets):
    
            d_features = cuda.to_device(features)
            d_targets = cuda.to_device(targets)
            d_weights = cuda.to_device(np.ones(num_weights, dtype=np.longdouble))
            d_partials = cuda.to_device(np.zeros(num_weights, dtype=np.longdouble))
    
            block_costs = np.zeros(num_blocks, dtype=np.float64)
            d_block_costs = cuda.to_device(block_costs)
    
    
            start = time()
    
            this.costs_func[num_blocks, threads_per_block](d_features, d_targets, d_weights, d_block_costs, d_partials)
    
            mean_cost = d_block_costs.copy_to_host().sum() / num_blocks
    
            elapsedtime = time() - start
            print("Elapsed:", elapsedtime)
            print("Mean cost:", mean_cost)
            print("Partials:", d_partials.copy_to_host())
            input("Done")
    
        @cuda.jit
        def costs_func(d_features, d_targets, d_weights, d_block_costs, d_partials):
    
            threadIdx = cuda.threadIdx.x
    
            s_features = cuda.shared.array((examples_per_block, num_weights), float64)
            s_targets = cuda.shared.array(examples_per_block, float64)
            s_weights = cuda.shared.array(num_weights, float64)
            s_partials = cuda.shared.array(num_weights, float64)
            s_costs = cuda.shared.array(examples_per_block, float64)
            s_initialcost = cuda.shared.array(1, float64)
    
            offsetx = cuda.blockIdx.x * cuda.blockDim.x
            for j in range(num_weights):
                s_features[threadIdx, j] = d_features[offsetx + threadIdx, j]
    
            s_targets[threadIdx] = d_targets[offsetx + threadIdx]
    
            cuda.syncthreads()
    
            if threadIdx == 0:  # init
                for i in range(num_weights):
                    s_weights[i] = d_weights[i]
    
            cuda.syncthreads()
    
            getCost(s_features, s_weights, s_targets, s_costs, s_initialcost)
    
    
    descend(features, targets)

Error without print:

	Traceback (most recent call last):
	  File "C:/Users/Joseph/PycharmProjects/Regression/bug.py", line 107, in <module>
		descend(features, targets)
	  File "C:/Users/Joseph/PycharmProjects/Regression/bug.py", line 70, in __new__
		mean_cost = d_block_costs.copy_to_host().sum() / num_blocks
	  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\devicearray.py", line 198, in copy_to_host
		_driver.device_to_host(hostary, self, self.alloc_size, stream=stream)
	  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 1598, in device_to_host
		fn(host_pointer(dst), device_pointer(src), size, *varargs)
	  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 288, in safe_cuda_api_call
		self._check_error(fname, retcode)
	  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 323, in _check_error
		raise CudaAPIError(retcode, msg)
	numba.cuda.cudadrv.driver.CudaAPIError: [716] Call to cuMemcpyDtoH results in UNKNOWN_CUDA_ERROR


Discussion:
This bug also occurs on the latest release, 0.33.0.
After moving some code around, I think I located the/a related issue. Setting s_initialcost shared array on line 87 to exactly 1, 3, 5 or 7 elements results in CUDA_UNKNOWN_ERROR.

    import numpy as np
    
    features = np.ones((20736, 2), dtype=np.float64)
    targets = np.ones(20736, dtype=np.float64) + np.e + 1
    
    ###Numba###
    num_examples, num_weights = features.shape
    
    elements_per_example = num_weights + 1
    bytes_per_element = 8
    
    num_SMPs = 12
    num_blocks = num_SMPs * 4
    max_threads = min(2048 * num_SMPs, 1024 * num_blocks)
    
    shared_memory_per_SMP = 48 * 1024
    shared_memory_per_block = int(
        min(shared_memory_per_SMP, shared_memory_per_SMP * num_SMPs / num_blocks) * 0.01)  # Use 1% of available memory
    
    weights_memory = num_weights * bytes_per_element
    
    max_elements_per_block = int((shared_memory_per_block - weights_memory) // bytes_per_element)
    
    examples_per_block = int(max_elements_per_block // elements_per_example)
    
    threads_per_block = examples_per_block
    
    print("num_weights:", num_weights)
    print("examples_per_block:", examples_per_block)
    print("threads_per_block:", threads_per_block)
    print("num_blocks:", num_blocks)
    ###Numba###
    
    from numba import cuda, float64, vectorize
    from time import time
    
    minimum_delta = 1e-10
    MAX_EXPONENT = int(round(np.log(1000 / minimum_delta) / np.log(3)))
    
    
    @cuda.jit(device=True)
    def getCost(s_features, s_weights, s_targets, s_costs, s_initialcost):
        threadIdx = cuda.threadIdx.x
    
        prediction = 0
        for j in range(num_weights):
            prediction += s_features[threadIdx,j] * s_weights[j]
        s_costs[threadIdx] = abs(prediction - s_targets[threadIdx])
    
        if threadIdx == 0:
            cost = 0
            for i in range(examples_per_block):
                cost += s_costs[i]
            s_initialcost[0] = cost
        cuda.syncthreads()
    
    class descend:
        def __new__(this, features, targets):
    
            d_features = cuda.to_device(features)
            d_targets = cuda.to_device(targets)
            d_weights = cuda.to_device(np.ones(num_weights, dtype=np.longdouble))
            d_partials = cuda.to_device(np.zeros(num_weights, dtype=np.longdouble))
    
            block_costs = np.zeros(num_blocks, dtype=np.float64)
            d_block_costs = cuda.to_device(block_costs)
    
            start = time()
    
            this.costs_func[num_blocks, threads_per_block](d_features, d_targets, d_weights, d_block_costs, d_partials)
    
            elapsedtime = time() - start
            print("Elapsed:", elapsedtime)
            print(d_features.copy_to_host())
            print(d_targets.copy_to_host())
            print(d_block_costs.copy_to_host()/examples_per_block)
            print(d_weights.copy_to_host())
            print(d_partials.copy_to_host())
    
        @cuda.jit
        def costs_func(d_features, d_targets, d_weights, d_block_costs, d_partials):
    
            threadIdx = cuda.threadIdx.x
    
            s_features = cuda.shared.array((examples_per_block, num_weights), float64)
            s_targets = cuda.shared.array(examples_per_block, float64)
            s_initialcost = cuda.shared.array(7, float64) # Bug
            s_weights = cuda.shared.array(num_weights, float64)
            s_partials = cuda.shared.array(num_weights, float64)
            s_block_costs = cuda.shared.array(num_blocks, float64)
            s_costs = cuda.shared.array(examples_per_block, float64)
    
    
            offsetx = cuda.blockIdx.x * cuda.blockDim.x
            for j in range(num_weights):
                s_features[threadIdx, j] = d_features[offsetx + threadIdx, j]
    
            s_targets[threadIdx] = d_targets[offsetx + threadIdx]
    
            cuda.syncthreads()
    
            if threadIdx == 0:  # init
                for i in range(num_weights):
                    s_weights[i] = d_weights[i]
    
            cuda.syncthreads()
    
            getCost(s_features, s_weights, s_targets, s_costs, s_initialcost)
    
            for i in range(examples_per_block):
                for j in range(num_weights):
                    s_features[i, j] = 1330
                    d_features[i, j] = s_features[i, j]
    
                s_targets[i] = 1331
                d_targets[i] = s_targets[i]
    
            blockcost_rs = 0
            for i in range(examples_per_block):
                blockcost_rs += s_costs[i]
    
            s_block_costs[cuda.blockIdx.x] = blockcost_rs
    
            d_block_costs[cuda.blockIdx.x] = s_block_costs[cuda.blockIdx.x]
    
            for j in range(num_weights):
                s_weights[j] = 1333
                d_weights[j] = s_weights[j]
                s_partials[j] = 1334
                d_partials[j] = s_partials[j]
    
            d_block_costs[0] = s_initialcost[0] * 10
    
    descend(features, targets)
Have you tried running the code with the `cuda-memcheck` tool?
This is the output:

```
(C:\Program Files\Anaconda3) C:\cooda\bin>cuda-memcheck --tool memcheck python bug.py
========= CUDA-MEMCHECK
num_weights: 2
examples_per_block: 4
threads_per_block: 4
num_blocks: 48
Elapsed: 0.4427809715270996
Traceback (most recent call last):
  File "C:\Users\Joseph\Desktop\bug.py", line 134, in <module>
    descend(features, targets)
  File "C:\Users\Joseph\Desktop\bug.py", line 74, in __new__
    print(d_features.copy_to_host())
  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\devicearray.py", line 198, in copy_to_host
    _driver.device_to_host(hostary, self, self.alloc_size, stream=stream)
  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 1598, in device_to_host
    fn(host_pointer(dst), device_pointer(src), size, *varargs)
  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 288, in safe_cuda_api_call
    self._check_error(fname, retcode)
  File "C:\Program Files\Anaconda3\lib\site-packages\numba\cuda\cudadrv\driver.py", line 323, in _check_error
    raise CudaAPIError(retcode, msg)
numba.cuda.cudadrv.driver.CudaAPIError: [716] Call to cuMemcpyDtoH results in UNKNOWN_CUDA_ERROR
========= ERROR SUMMARY: 0 errors
```

If you want to test the code yourself just replace "for in" with "for j in"; for some reason Github is messing with the code.
I ran this a few times, seems like sometimes it's fine and sometimes I get:
```
========= Invalid __shared__ read of size 16
=========     at 0x00000448 in cudapy::__main__::descend::costs_func$241(Array<double, int=2, A, mutable, aligned>, Array<double, int=1, A, mutable, aligned>, Array<double, int=1, A, mutable, aligned>, Array<double, int=1, A, mutable, aligned>, Array<double, int=1, A, mutable, aligned>)
=========     by thread (3,0,0) in block (45,0,0)
=========     Address 0x000000a8 is misaligned
=========     Saved host backtrace up to driver entry point at kernel launch time
=========     Host Frame:libcuda.so (cuLaunchKernel + 0x2c5) [0x204505]
```
many times, all relating to addressing the `s_initialcost` array, and then the `UNKNOWN_CUDA_ERROR`. I'll try and have a better look at it a bit later.
Interestingly, the read size is 16 bytes on memory of double (8 byte) type.  The compiler has optimized the instructions to do a vector load for double2.

The PTX shows:

```
ld.shared.v2.f64 	{%fd1, %fd2}, [%rd10];
```

which originates from `s_features[threadIdx,j]` in the loop:

```python
    for j in range(num_weights):
        prediction += s_features[threadIdx,j] * s_weights[j]
```

The compiler knows `num_weights` is a constant value `2`.  This enables unrolling and vectorization to form the vector load only if the `s_features` is aligned to 16-byte.  The inst `ld.shared.v2.f64` requires the address be naturally aligned to the type, which is a 16-byte double2.   However, `s_features` is declared to align to 8-byte.  Perhaps, the start of each shared memory is guaranteed to fall into a bigger alignment (e.g. 32-byte).  The vectorization would be valid in that case.

Then, why would changing the size of `s_initialcost` affect the load on `s_features`?  Changing `s_initialcost` to any odd number seems to cause the problem.  Any even number is fine though.

One possible reason is that when more than one block is scheduled to the multprocessor.  The shared memory for each block is append after each other.  If the shared memory size ends at 0x8, the shared memory of the next block starts at 0x8, which satisfy the alignment for `s_features` but not for the vector load.


Resolving patch:
diff --git a/numba/cuda/cudaimpl.py b/numba/cuda/cudaimpl.py
--- a/numba/cuda/cudaimpl.py
+++ b/numba/cuda/cudaimpl.py
@@ -371,6 +371,8 @@ def _generic_array(context, builder, shape, dtype, symbol_name, addrspace,
 
         # Create global variable in the requested address-space
         gvmem = lmod.add_global_variable(laryty, symbol_name, addrspace)
+        # Specify alignment to avoid misalignment bug
+        gvmem.align = context.get_abi_sizeof(lldtype)
 
         if elemcount <= 0:
             if can_dynsized:    # dynamic shared memory


Test patch:
diff --git a/numba/cuda/tests/cudapy/test_sm.py b/numba/cuda/tests/cudapy/test_sm.py
--- a/numba/cuda/tests/cudapy/test_sm.py
+++ b/numba/cuda/tests/cudapy/test_sm.py
@@ -1,9 +1,10 @@
-from numba import cuda, int32
+from numba import cuda, int32, float64
 
 from numba.cuda.testing import unittest
 
 import numpy as np
 
+
 class TestSharedMemoryIssue(unittest.TestCase):
     def test_issue_953_sm_linkage_conflict(self):
         @cuda.jit(device=True)
@@ -36,6 +37,37 @@ def test_issue_1051_shared_size_broken_2d(self):
     def test_issue_1051_shared_size_broken_3d(self):
         self._check_shared_array_size((2, 3, 4), 24)
 
+    def test_issue_2393(self):
+        """
+        Test issue of warp misalign address due to nvvm not knowing the
+        alignment(? but it should have taken the natural alignment of the type)
+        """
+        num_weights = 2
+        num_blocks = 48
+        examples_per_block = 4
+        threads_per_block = 1
+
+        @cuda.jit
+        def costs_func(d_block_costs):
+            s_features = cuda.shared.array((examples_per_block, num_weights),
+                                           float64)
+            s_initialcost = cuda.shared.array(7, float64)  # Bug
+
+            threadIdx = cuda.threadIdx.x
+
+            prediction = 0
+            for j in range(num_weights):
+                prediction += s_features[threadIdx, j]
+
+            d_block_costs[0] = s_initialcost[0] + prediction
+
+        block_costs = np.zeros(num_blocks, dtype=np.float64)
+        d_block_costs = cuda.to_device(block_costs)
+
+        costs_func[num_blocks, threads_per_block](d_block_costs)
+
+        cuda.synchronize()
+
 
 if __name__ == '__main__':
     unittest.main()