NagaYu/litmus-kernels huggingface dataset:
Kernel: layernorm_race_stats
Family: layernorm
Description: per-row statistics published through a shared scratch cell
Bug class: race
Correct: no
Numerically observable: yes
Why the standard test misses it: rows of i.i.d. noise have near-identical statistics, so reading another row's mean and variance barely changes the output
Standard allclose test detects it: yes
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK = 32
# litmus: buffer x_ptr = M*S
# litmus: buffer w_ptr = N
# litmus: buffer b_ptr = N
# litmus: buffer out_ptr = M*N
# litmus: buffer stats_ptr = 2
# litmus: grid = M
# litmus: subst S = N + P
# litmus: sym P in [0, 8]

@triton.jit
def layernorm_race(x_ptr, w_ptr, b_ptr, out_ptr, stats_ptr, M, N, S, BLOCK: tl.constexpr):
    row = tl.program_id(0)
    s1 = 0.0
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * S + cols, mask=mask, other=0.0)
        s1 += tl.sum(v, axis=0)
    mean = s1 / N
    s2 = 0.0
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * S + cols, mask=mask, other=0.0)
        d = tl.where(mask, v - mean, 0.0)
        s2 += tl.sum(d * d, axis=0)
    var = s2 / N
    tl.store(stats_ptr + 0, mean)
    tl.store(stats_ptr + 1, var)
    mean = tl.load(stats_ptr + 0)
    var = tl.load(stats_ptr + 1)
    rstd = 1.0 / tl.sqrt(var + 1e-5)
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * S + cols, mask=mask, other=0.0)
        w = tl.load(w_ptr + cols, mask=mask, other=0.0)
        b = tl.load(b_ptr + cols, mask=mask, other=0.0)
        tl.store(out_ptr + row * N + cols, (v - mean) * rstd * w + b, mask=mask)

Dims: M=8, N=512, S=512
Blocks: BLOCK=32
Dtype: float32 (accumulation dtype: float32)
Input domain: x ∈ [-10, 10], w ∈ [-10, 10], b ∈ [-10, 10]
Kernel: row_sum_barrier
Family: reduction
Description: tl.debug_barrier() under a lane-dependent condition
Bug class: race
Correct: no
Numerically observable: no
Why the standard test misses it: divergent barriers produce no numerical error at all until the schedule changes -- they are invisible to any output comparison
Standard allclose test detects it: no
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK = 32
# litmus: buffer x_ptr = M*S
# litmus: buffer out_ptr = M
# litmus: grid = M
# litmus: subst S = N + P
# litmus: sym P in [0, 8]

@triton.jit
def row_sum_barrier(x_ptr, out_ptr, M, N, S, BLOCK: tl.constexpr):
    row = tl.program_id(0)
    acc = tl.zeros((BLOCK,), dtype=tl.float32)
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * S + cols, mask=mask, other=0.0)
        acc += v
    if tl.arange(0, BLOCK)[0] < N:
        tl.debug_barrier()
    tl.store(out_ptr + row, tl.sum(acc, axis=0))

Dims: M=8, N=512, S=512
Blocks: BLOCK=32
Dtype: float32 (accumulation dtype: float32)
Input domain: x ∈ [-1000, 1000]
Kernel: matmul_fp16_acc16
Family: matmul
Description: fp16 inputs accumulated in fp16 instead of fp32
Bug class: dtype_promotion
Correct: no
Numerically observable: yes
Why the standard test misses it: at K=512 with values in [0,1) the fp16 accumulator has not yet stagnated; the error stays under rtol=1e-2
Standard allclose test detects it: no
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK_M = 32
# litmus: constexpr BLOCK_N = 32
# litmus: constexpr BLOCK_K = 32
# litmus: buffer a_ptr = M*K
# litmus: buffer b_ptr = K*N
# litmus: buffer c_ptr = M*N
# litmus: grid = cdiv(M, BLOCK_M), cdiv(N, BLOCK_N)

@triton.jit
def matmul(a_ptr, b_ptr, c_ptr, M, N, K,
           BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):
    pm = tl.program_id(0)
    pn = tl.program_id(1)
    offs_m = pm * BLOCK_M + tl.arange(0, BLOCK_M)
    offs_n = pn * BLOCK_N + tl.arange(0, BLOCK_N)
    acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float16)
    for k in range(0, tl.cdiv(K, BLOCK_K)):
        offs_k = k * BLOCK_K + tl.arange(0, BLOCK_K)
        a = tl.load(a_ptr + offs_m[:, None] * K + offs_k[None, :],
                    mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)
        b = tl.load(b_ptr + offs_k[:, None] * N + offs_n[None, :],
                    mask=(offs_k[:, None] < K) & (offs_n[None, :] < N), other=0.0)
        acc += tl.dot(a, b)
    tl.store(c_ptr + offs_m[:, None] * N + offs_n[None, :], acc,
             mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))

Dims: M=128, N=128, K=512
Blocks: BLOCK_M=32, BLOCK_N=32, BLOCK_K=32
Dtype: float16 (accumulation dtype: float32)
Input domain: a ∈ [-4, 4], b ∈ [-4, 4]
Kernel: row_sum_fp16_acc
Family: reduction
Description: row sum accumulated in fp16
Bug class: dtype_promotion
Correct: no
Numerically observable: yes
Why the standard test misses it: an fp16 accumulator stagnates above ~2048 and overflows above 65504; with N=512 and values in [0,1) the running total only reaches about 256
Standard allclose test detects it: no
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK = 32
# litmus: buffer x_ptr = M*S
# litmus: buffer out_ptr = M
# litmus: grid = M
# litmus: subst S = N + P
# litmus: sym P in [0, 8]

@triton.jit
def row_reduce(x_ptr, out_ptr, M, N, S, BLOCK: tl.constexpr):
    row = tl.program_id(0)
    acc = tl.zeros((BLOCK,), dtype=tl.float16)
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * S + cols, mask=mask, other=0.0)
        acc += v
    tl.store(out_ptr + row, tl.sum(acc, axis=0))

Dims: M=8, N=512, S=512
Blocks: BLOCK=32
Dtype: float16 (accumulation dtype: float32)
Input domain: x ∈ [-4, 4]
Kernel: row_sum_wrong_stride
Family: reduction
Description: row base computed with N instead of the row stride S
Bug class: stride
Correct: no
Numerically observable: yes
Why the standard test misses it: benchmarks always pass contiguous tensors, where the row stride equals the row width and the bug is a no-op
Standard allclose test detects it: no
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK = 32
# litmus: buffer x_ptr = M*S
# litmus: buffer out_ptr = M
# litmus: grid = M
# litmus: subst S = N + P
# litmus: sym P in [0, 8]

@triton.jit
def row_reduce(x_ptr, out_ptr, M, N, S, BLOCK: tl.constexpr):
    row = tl.program_id(0)
    acc = tl.zeros((BLOCK,), dtype=tl.float32)
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * N + cols, mask=mask, other=0.0)
        acc += v
    tl.store(out_ptr + row, tl.sum(acc, axis=0))

Dims: M=8, N=512, S=512
Blocks: BLOCK=32
Dtype: float32 (accumulation dtype: float32)
Input domain: x ∈ [-1000, 1000]
Kernel: layernorm_wrong_stride
Family: layernorm
Description: layernorm reads rows with stride N on a padded tensor
Bug class: stride
Correct: no
Numerically observable: yes
Why the standard test misses it: benchmarks always pass contiguous tensors, where the row stride equals the row width and the bug is a no-op
Standard allclose test detects it: no
Litmus verdict: reject

import triton
import triton.language as tl

# litmus: constexpr BLOCK = 32
# litmus: buffer x_ptr = M*S
# litmus: buffer w_ptr = N
# litmus: buffer b_ptr = N
# litmus: buffer out_ptr = M*N
# litmus: grid = M
# litmus: subst S = N + P
# litmus: sym P in [0, 8]

@triton.jit
def layernorm(x_ptr, w_ptr, b_ptr, out_ptr, M, N, S, BLOCK: tl.constexpr):
    row = tl.program_id(0)
    s1 = 0.0
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * N + cols, mask=mask, other=0.0)
        s1 += tl.sum(v, axis=0)
    mean = s1 / N
    s2 = 0.0
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * N + cols, mask=mask, other=0.0)
        d = tl.where(mask, v - mean, 0.0)
        s2 += tl.sum(d * d, axis=0)
    var = s2 / N
    rstd = 1.0 / tl.sqrt(var + 1e-5)
    for start in range(0, N, BLOCK):
        cols = start + tl.arange(0, BLOCK)
        mask = cols < N
        v = tl.load(x_ptr + row * N + cols, mask=mask, other=0.0)
        w = tl.load(w_ptr + cols, mask=mask, other=0.0)
        b = tl.load(b_ptr + cols, mask=mask, other=0.0)
        tl.store(out_ptr + row * N + cols, (v - mean) * rstd * w + b, mask=mask)

Dims: M=8, N=512, S=512
Blocks: BLOCK=32
Dtype: float32 (accumulation dtype: float32)
Input domain: x ∈ [-10, 10], w ∈ [-10, 10], b ∈ [-10, 10]