Custom CUDA kernel very slow for half precision

F-Schlatt: <p>I have the following (severely unoptimized) CUDA kernel, which takes two batched matrices and does a windowed matrix multiplication between the two. It’s not super pretty, but works (hints for how to optimize the kernel are also appreciated). However, for half precision fp16 tensors, the kernel is very slow (100x slower on an A100 GPU; see the test script and output below). I’ve found a GitHub issue, stating that <code>fastAtomicAdd</code> is more efficient than <code>gpuAtomicAdd</code>, I’m unable to compile the kernel with fastAtomicAdd and pytorch 1.13 because <code>fastAtomicAdd</code> is not defined. If anyone knows why the kernel is so much slower for half precision and could give me a hint, that would be greatly appreciated.</p>
<pre><code class="lang-auto">#include &lt;torch/extension.h&gt;
#include &lt;ATen/cuda/Atomic.cuh&gt;

#include "utils.cuh"
#include &lt;algorithm&gt;
#include &lt;cuda.h&gt;
#include &lt;cuda_runtime.h&gt;

#define THREADS 512
#define BLOCKS(N) (N + THREADS - 1) / THREADS

template &lt;typename scalar_t&gt;
__global__ void window_matmul_fw_cuda_kernel(
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; other_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; out_accessor,
    int window_size,
    int batch_size,
    int seq_len,
    int hidden_dim)
{
  int thread_idx = blockIdx.x * blockDim.x + threadIdx.x;
  int h, w, s, b, w_idx, full_window_size;
  full_window_size = window_size * 2 + 1;
  if (thread_idx &gt;= batch_size * seq_len * full_window_size * hidden_dim)
  {
    return;
  }
  b = thread_idx / hidden_dim / full_window_size / seq_len % batch_size;
  s = thread_idx / hidden_dim / full_window_size % seq_len;
  w = s - window_size + (thread_idx / hidden_dim % full_window_size);
  if (w &lt; 0 || w &gt;= seq_len)
    return;
  h = thread_idx % hidden_dim;
  w_idx = w - std::min(0, s - window_size) - std::max(0, s - window_size);
  gpuAtomicAddNoReturn(&amp;out_accessor[b][s][w_idx], src_accessor[b][s][h] * other_accessor[b][h][w]);
}

torch::Tensor window_matmul_fw_cuda(torch::Tensor src, torch::Tensor other, int window_size)
{
  CHECK_CUDA(src);
  CHECK_CUDA(other);

  CHECK_INPUT(src.dim() == other.dim());
  CHECK_INPUT(src.size(0) == src.size(0));
  CHECK_INPUT(src.size(-1) == other.size(-2));
  CHECK_INPUT(src.size(-2) == other.size(-1));

  src = src.contiguous();
  other = other.contiguous();

  torch::Tensor out;
  auto sizes = src.sizes().vec();
  sizes[2] = window_size * 2 + 1;
  out = torch::zeros(sizes, src.options());
  int batch_size, seq_len, hidden_dim;
  batch_size = src.size(0);
  seq_len = src.size(1);
  hidden_dim = src.size(2);

  AT_DISPATCH_FLOATING_TYPES_AND_HALF(
      out.scalar_type(), "window_matmul_fw_cuda", [&amp;]
      {
        auto src_accessor = src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto other_accessor = other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto out_accessor = out.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        window_matmul_fw_cuda_kernel&lt;scalar_t&gt;&lt;&lt;&lt;BLOCKS(batch_size*seq_len*(window_size * 2 + 1)*hidden_dim),THREADS&gt;&gt;&gt;(
          src_accessor,
          other_accessor,
          out_accessor,
          window_size,
          batch_size,
          seq_len,
          hidden_dim
          ); });
  return out;
}
</code></pre>
<pre><code class="lang-auto">import torch

from window_matmul import window_matmul

BATCH_SIZE = 64
SEQ_LEN = 512
HIDDEN_DIM = 64
WINDOW_SIZE = 16

mat_1 = torch.randn(BATCH_SIZE, SEQ_LEN, HIDDEN_DIM).to("cuda")
mat_2 = torch.randn(BATCH_SIZE, SEQ_LEN, HIDDEN_DIM).to("cuda")


for dtype in [torch.float16, torch.float32]:
    mat_1 = mat_1.to(dtype)
    mat_2 = mat_2.to(dtype)
    with torch.no_grad():
        with torch.profiler.profile(
            activities=[torch.profiler.ProfilerActivity.CUDA]
        ) as prof:
            mat_3 = window_matmul(mat_1, mat_2, window_size=WINDOW_SIZE)
    print("#" * 20 + f" {dtype} " + "#" * 20)
    print(
        prof.key_averages(group_by_input_shape=True).table(
            sort_by="cuda_time_total", row_limit=10
        )
    )
</code></pre>
<pre><code class="lang-bash">$ python test.py 
STAGE:2023-01-09 21:25:24 1706475:1706475 ActivityProfilerController.cpp:294] Completed Stage: Warm Up
STAGE:2023-01-09 21:25:25 1706475:1706475 ActivityProfilerController.cpp:300] Completed Stage: Collection
#################### torch.float16 ####################
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
void window_matmul_fw_cuda_kernel&lt;c10::Half&gt;(at::Gen...         0.00%       0.000us         0.00%       0.000us       0.000us     697.518ms       100.00%     697.518ms     697.518ms             1  
void at::native::vectorized_elementwise_kernel&lt;4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       8.000us         0.00%       8.000us       8.000us             1  
                                       cudaLaunchKernel         0.01%      57.000us         0.01%      57.000us      28.500us       0.000us         0.00%       0.000us       0.000us             2  
                                  cudaDeviceSynchronize        99.99%     697.393ms        99.99%     697.393ms     697.393ms       0.000us         0.00%       0.000us       0.000us             1  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
Self CPU time total: 697.450ms
Self CUDA time total: 697.526ms

STAGE:2023-01-09 21:25:25 1706475:1706475 ActivityProfilerController.cpp:294] Completed Stage: Warm Up
STAGE:2023-01-09 21:25:25 1706475:1706475 ActivityProfilerController.cpp:300] Completed Stage: Collection
#################### torch.float32 ####################
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
void window_matmul_fw_cuda_kernel&lt;float&gt;(at::Generic...         0.00%       0.000us         0.00%       0.000us       0.000us       7.150ms        99.89%       7.150ms       7.150ms             1  
void at::native::vectorized_elementwise_kernel&lt;4, at...         0.00%       0.000us         0.00%       0.000us       0.000us       8.000us         0.11%       8.000us       8.000us             1  
                                       cudaLaunchKernel         0.31%      22.000us         0.31%      22.000us      11.000us       0.000us         0.00%       0.000us       0.000us             2  
                                  cudaDeviceSynchronize        99.69%       7.106ms        99.69%       7.106ms       7.106ms       0.000us         0.00%       0.000us       0.000us             1  
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  
Self CPU time total: 7.128ms
Self CUDA time total: 7.158ms

</code></pre>

F-Schlatt: <p><code>fastAtomicAdd</code> was the solution in the end. I was just not aware I had to import it from the <code>at::native</code>. For reference: here’s the updated code. One of the include statements was changed and <code>gpuAtomicAddNoReturn</code> was changed to <code>at::native::fastAtomicAdd</code></p>
<pre><code class="lang-auto">#include &lt;torch/extension.h&gt;
#include &lt;ATen/native/cuda/KernelUtils.cuh&gt;

#include "utils.cuh"
#include &lt;algorithm&gt;
#include &lt;cuda.h&gt;
#include &lt;cuda_runtime.h&gt;

#define THREADS 256
#define BLOCKS(N) (N + THREADS - 1) / THREADS

template &lt;typename scalar_t&gt;
__global__ void window_matmul_fw_cuda_kernel(
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; other_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; out_accessor,
    int window_size,
    int batch_size,
    int seq_len,
    int hidden_dim,
    int out_numel)
{
  int thread_idx = blockIdx.x * blockDim.x + threadIdx.x;
  int h, w, s, b, w_idx, full_window_size, index;
  full_window_size = window_size * 2 + 1;
  if (thread_idx &gt;= batch_size * seq_len * full_window_size * hidden_dim)
    return;
  b = thread_idx / hidden_dim / full_window_size / seq_len % batch_size;
  s = thread_idx / hidden_dim / full_window_size % seq_len;
  w = s - window_size + (thread_idx / hidden_dim % full_window_size);
  if (w &lt; 0 || w &gt;= seq_len)
    return;
  h = thread_idx % hidden_dim;
  w_idx = w - std::min(0, s - window_size) - std::max(0, s - window_size);
  index = b * out_accessor.stride(0) + s * out_accessor.stride(1) + w_idx * out_accessor.stride(2);
  at::native::fastAtomicAdd(out_accessor.data(), index, out_numel, src_accessor[b][s][h] * other_accessor[b][h][w], true);
}

torch::Tensor window_matmul_fw_cuda(torch::Tensor src, torch::Tensor other, int window_size)
{
  CHECK_CUDA(src);
  CHECK_CUDA(other);

  CHECK_INPUT(src.dim() == other.dim());
  CHECK_INPUT(src.size(0) == src.size(0));
  CHECK_INPUT(src.size(-1) == other.size(-2));
  CHECK_INPUT(src.size(-2) == other.size(-1));

  src = src.contiguous();
  other = other.contiguous();

  torch::Tensor out;
  auto sizes = src.sizes().vec();
  sizes[2] = window_size * 2 + 1;
  out = torch::zeros(sizes, src.options());
  int batch_size, seq_len, hidden_dim;
  batch_size = src.size(0);
  seq_len = src.size(1);
  hidden_dim = src.size(2);

  AT_DISPATCH_FLOATING_TYPES_AND_HALF(
      out.scalar_type(), "window_matmul_fw_cuda", [&amp;]
      {
        auto src_accessor = src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto other_accessor = other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto out_accessor = out.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        window_matmul_fw_cuda_kernel&lt;scalar_t&gt;&lt;&lt;&lt;BLOCKS(batch_size*seq_len*(window_size * 2 + 1)*hidden_dim),THREADS&gt;&gt;&gt;(
          src_accessor,
          other_accessor,
          out_accessor,
          window_size,
          batch_size,
          seq_len,
          hidden_dim,
          out.numel()
          ); });
  return out;
}

template &lt;typename scalar_t&gt;
__global__ void window_matmul_bw_cuda_kernel(
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; other_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_output_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_other_accessor,
    int window_size,
    int batch_size,
    int seq_len,
    int hidden_dim,
    int grad_src_numel,
    int grad_other_numel)
{
  int thread_idx = blockIdx.x * blockDim.x + threadIdx.x;
  int h, w, s, b, w_idx, full_window_size, index;
  full_window_size = window_size * 2 + 1;
  if (thread_idx &gt;= batch_size * seq_len * full_window_size * hidden_dim)
    return;
  b = thread_idx / hidden_dim / full_window_size / seq_len % batch_size;
  s = thread_idx / hidden_dim / full_window_size % seq_len;
  w = s - window_size + (thread_idx / hidden_dim % full_window_size);
  if (w &lt; 0 || w &gt;= seq_len)
    return;
  h = thread_idx % hidden_dim;
  w_idx = w - std::min(0, s - window_size) - std::max(0, s - window_size);
  index = b * grad_src_accessor.stride(0) + s * grad_src_accessor.stride(1) + h * grad_src_accessor.stride(2);
  at::native::fastAtomicAdd(grad_src_accessor.data(), index, grad_src_numel, other_accessor[b][h][w] * grad_output_accessor[b][s][w_idx], true);
  index = b * grad_other_accessor.stride(0) + h * grad_other_accessor.stride(1) + w * grad_other_accessor.stride(2);
  at::native::fastAtomicAdd(grad_other_accessor.data(), index, grad_other_numel, src_accessor[b][s][h] * grad_output_accessor[b][s][w_idx], true);
}

std::tuple&lt;torch::Tensor, torch::Tensor&gt; window_matmul_bw_cuda(
    torch::Tensor src, torch::Tensor other, int window_size, torch::Tensor grad_output)
{
  CHECK_CUDA(src);
  CHECK_CUDA(other);

  CHECK_INPUT(src.dim() == other.dim());
  CHECK_INPUT(src.size(0) == src.size(0));
  CHECK_INPUT(src.size(-1) == other.size(-2));
  CHECK_INPUT(src.size(-2) == other.size(-1));

  src = src.contiguous();
  other = other.contiguous();
  grad_output = grad_output.contiguous();

  torch::Tensor grad_src, grad_other;
  grad_src = torch::zeros(src.sizes().vec(), src.options());
  grad_other = torch::zeros(other.sizes().vec(), other.options());
  int batch_size, seq_len, hidden_dim;
  batch_size = src.size(0);
  seq_len = src.size(1);
  hidden_dim = src.size(2);

  AT_DISPATCH_FLOATING_TYPES_AND_HALF(
      grad_src.scalar_type(), "window_matmul_bw_cuda", [&amp;]
      {
        auto src_accessor = src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto other_accessor = other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_output_accessor = grad_output.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_src_accessor = grad_src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_other_accessor = grad_other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        window_matmul_bw_cuda_kernel&lt;scalar_t&gt;&lt;&lt;&lt;BLOCKS(batch_size*seq_len*(window_size * 2 + 1)*hidden_dim),THREADS&gt;&gt;&gt;(
          src_accessor,
          other_accessor,
          grad_output_accessor,
          grad_src_accessor,
          grad_other_accessor,
          window_size,
          batch_size,
          seq_len,
          hidden_dim,
          grad_src.numel(),
          grad_other.numel()
          ); });
  return std::make_tuple(grad_src, grad_other);
}

template &lt;typename scalar_t&gt;
__global__ void unwindow_matmul_fw_cuda_kernel(
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; other_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; out_accessor,
    int window_size,
    int batch_size,
    int seq_len,
    int hidden_dim,
    int out_numel)
{
  int thread_idx = blockIdx.x * blockDim.x + threadIdx.x;
  int h, w, s, b, w_idx, full_window_size, index;
  full_window_size = window_size * 2 + 1;
  if (thread_idx &gt;= batch_size * seq_len * full_window_size * hidden_dim)
    return;
  b = thread_idx / hidden_dim / full_window_size / seq_len % batch_size;
  s = thread_idx / hidden_dim / full_window_size % seq_len;
  w = s - window_size + (thread_idx / hidden_dim % full_window_size);
  if (w &lt; 0 || w &gt;= seq_len)
    return;
  h = thread_idx % hidden_dim;
  w_idx = w - std::min(0, s - window_size) - std::max(0, s - window_size);
  index = b * out_accessor.stride(0) + s * out_accessor.stride(1) + h * out_accessor.stride(2);
  at::native::fastAtomicAdd(out_accessor.data(), index, out_numel, src_accessor[b][s][w_idx] * other_accessor[b][w][h], true);
}

torch::Tensor unwindow_matmul_fw_cuda(torch::Tensor src, torch::Tensor other, int window_size)
{
  CHECK_CUDA(src);
  CHECK_CUDA(other);

  CHECK_INPUT(src.dim() == other.dim());
  CHECK_INPUT(src.size(0) == src.size(0));
  CHECK_INPUT(src.size(-1) == window_size * 2 + 1);
  CHECK_INPUT(src.size(-2) == other.size(-2));

  src = src.contiguous();
  other = other.contiguous();

  torch::Tensor out;
  auto sizes = other.sizes().vec();
  out = torch::zeros(sizes, src.options());
  int batch_size, seq_len, hidden_dim;
  batch_size = other.size(0);
  seq_len = other.size(1);
  hidden_dim = other.size(2);

  AT_DISPATCH_FLOATING_TYPES_AND_HALF(
      out.scalar_type(), "unwindow_matmul_fw_cuda", [&amp;]
      {
        auto src_accessor = src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto other_accessor = other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto out_accessor = out.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();

        unwindow_matmul_fw_cuda_kernel&lt;scalar_t&gt;&lt;&lt;&lt;BLOCKS(batch_size*seq_len*(window_size * 2 + 1)*hidden_dim),THREADS&gt;&gt;&gt;(
          src_accessor,
          other_accessor,
          out_accessor,
          window_size,
          batch_size,
          seq_len,
          hidden_dim,
          out.numel()
          ); });
  return out;
}

template &lt;typename scalar_t&gt;
__global__ void unwindow_matmul_bw_cuda_kernel(
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; other_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_output_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_src_accessor,
    torch::PackedTensorAccessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt; grad_other_accessor,
    int window_size,
    int batch_size,
    int seq_len,
    int hidden_dim,
    int grad_src_numel,
    int grad_other_numel)
{
  int thread_idx = blockIdx.x * blockDim.x + threadIdx.x;
  int h, w, s, b, w_idx, full_window_size, index;
  full_window_size = window_size * 2 + 1;
  if (thread_idx &gt;= batch_size * seq_len * full_window_size * hidden_dim)
    return;
  b = thread_idx / hidden_dim / full_window_size / seq_len % batch_size;
  s = thread_idx / hidden_dim / full_window_size % seq_len;
  w = s - window_size + (thread_idx / hidden_dim % full_window_size);
  if (w &lt; 0 || w &gt;= seq_len)
    return;
  h = thread_idx % hidden_dim;
  w_idx = w - std::min(0, s - window_size) - std::max(0, s - window_size);
  index = b * grad_src_accessor.stride(0) + s * grad_src_accessor.stride(1) + w_idx * grad_src_accessor.stride(2);
  at::native::fastAtomicAdd(grad_src_accessor.data(), index, grad_src_numel, other_accessor[b][w][h] * grad_output_accessor[b][s][h], true);
  index = b * grad_other_accessor.stride(0) + w * grad_other_accessor.stride(1) + h * grad_other_accessor.stride(2);
  at::native::fastAtomicAdd(grad_other_accessor.data(), index, grad_other_numel, src_accessor[b][s][w_idx] * grad_output_accessor[b][s][h], true);
}

std::tuple&lt;torch::Tensor, torch::Tensor&gt; unwindow_matmul_bw_cuda(
    torch::Tensor src, torch::Tensor other, int window_size, torch::Tensor grad_output)
{
  CHECK_CUDA(src);
  CHECK_CUDA(other);

  CHECK_INPUT(src.dim() == other.dim());
  CHECK_INPUT(src.size(0) == src.size(0));
  CHECK_INPUT(src.size(-1) == window_size * 2 + 1);
  CHECK_INPUT(src.size(-2) == other.size(-2));

  src = src.contiguous();
  other = other.contiguous();
  grad_output = grad_output.contiguous();

  torch::Tensor grad_src, grad_other;
  grad_src = torch::zeros(src.sizes().vec(), src.options());
  grad_other = torch::zeros(other.sizes().vec(), other.options());
  int batch_size, seq_len, hidden_dim;
  batch_size = other.size(0);
  seq_len = other.size(1);
  hidden_dim = other.size(2);

  AT_DISPATCH_FLOATING_TYPES_AND_HALF(
      grad_src.scalar_type(), "unwindow_matmul_bw_cuda", [&amp;]
      {
        auto src_accessor = src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto other_accessor = other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_output_accessor = grad_output.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_src_accessor = grad_src.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();
        auto grad_other_accessor = grad_other.packed_accessor32&lt;scalar_t, 3, torch::RestrictPtrTraits&gt;();

        unwindow_matmul_bw_cuda_kernel&lt;scalar_t&gt;&lt;&lt;&lt;BLOCKS(batch_size*seq_len*(window_size * 2 + 1)*hidden_dim),THREADS&gt;&gt;&gt;(
          src_accessor,
          other_accessor,
          grad_output_accessor,
          grad_src_accessor,
          grad_other_accessor,
          window_size,
          batch_size,
          seq_len,
          hidden_dim,
          grad_src.numel(),
          grad_other.numel()
          ); });
  return std::make_tuple(grad_src, grad_other);
}
</code></pre>