jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_triton_cross_entropy(device, dtype):
    if IN_CI:
        pytest.skip("Currently these tests are skipped in CI for speed")

    logits = torch.randn([2048, 50257], device=device, dtype=dtype)
    labels = torch.randint(0, 50257, [2048], device=device)
    reduction = "sum"
    ignore_index = labels[5].item()
    weight = torch.rand(50257, device=device, dtype=dtype, requires_grad=False)
    expected = torch.nn.functional.cross_entropy(
        logits, labels, weight=weight, reduction=reduction, ignore_index=ignore_index
    )

    def test(logits, labels, weight, reduction, ignore_index):
        return thunder.torch.cross_entropy(
            logits, labels, weight=weight, reduction=reduction, ignore_index=ignore_index
        )

    ctest = thunder.jit(test, executors=[triton_ex])
    actual = ctest(logits, labels, weight, reduction, ignore_index)
    torch.testing.assert_allclose(actual, expected)
    last_trace = thunder.last_traces(ctest)[-1]
    assert any(bsym.sym.name == "triton_crossentropy" for bsym in last_trace.bound_symbols)
//  [python]
def test_triton_cross_entropy(device, dtype):
    if IN_CI:
        pytest.skip("Currently these tests are skipped in CI for speed")

    logits = torch.randn([2048, 50257], device=device, dtype=dtype)
    labels = torch.randint(0, 50257, [2048], device=device)
    reduction = "sum"
    ignore_index = labels[5].item()
    weight = torch.rand(50257, device=device, dtype=dtype, requires_grad=False)
    expected = torch.nn.functional.cross_entropy(
        logits, labels, weight=weight, reduction=reduction, ignore_index=ignore_index
    )

    def test(logits, labels, weight, reduction, ignore_index):
        return thunder.torch.cross_entropy(
            logits, labels, weight=weight, reduction=reduction, ignore_index=ignore_index
        )

    ctest = thunder.jit(test, executors=[triton_ex])
    actual = ctest(logits, labels, weight, reduction, ignore_index)
    torch.testing.assert_close(actual, expected)
    last_trace = thunder.last_traces(ctest)[-1]
    assert any(bsym.sym.name == "triton_crossentropy" for bsym in last_trace.bound_symbols)
//  [python]
def test_reorder_batched_ad_lengths(
        self,
        B: int,
        T: int,
        L: int,
        A: int,
        Dtype: torch.dtype,
        broadcast_lengths: bool,
    ) -> None:
        if broadcast_lengths:
            cat_ad_lengths = (
                torch.cat([torch.tensor([L for _ in range(T)]) for _ in range(B)], 0)
                .cuda()
                .to(Dtype)
            )
            cat_ad_lengths_broadcasted = cat_ad_lengths.tile([A])
        else:
            cat_ad_lengths = (
                torch.cat(
                    [torch.tensor([L for _ in range(T * A)]) for _ in range(B)], 0
                )
                .cuda()
                .to(Dtype)
            )
            cat_ad_lengths_broadcasted = cat_ad_lengths
        batch_offsets = torch.tensor([A * b for b in range(B + 1)]).int().cuda()
        num_ads_in_batch = B * A
        reordered_batched_ad_lengths = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths, batch_offsets, num_ads_in_batch, broadcast_lengths
        )
        torch.testing.assert_allclose(
            cat_ad_lengths_broadcasted, reordered_batched_ad_lengths
        )

        cat_ad_lengths_cpu = cat_ad_lengths.cpu()
        batch_offsets_cpu = batch_offsets.cpu()
        reordered_batched_ad_lengths_cpu = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths_cpu, batch_offsets_cpu, num_ads_in_batch, broadcast_lengths
        )
        torch.testing.assert_allclose(
            reordered_batched_ad_lengths_cpu, reordered_batched_ad_lengths.cpu()
        )
//  [python]
def test_reorder_batched_ad_lengths(
        self,
        B: int,
        T: int,
        L: int,
        A: int,
        Dtype: torch.dtype,
        broadcast_lengths: bool,
    ) -> None:
        if broadcast_lengths:
            cat_ad_lengths = (
                torch.cat([torch.tensor([L for _ in range(T)]) for _ in range(B)], 0)
                .cuda()
                .to(Dtype)
            )
            cat_ad_lengths_broadcasted = cat_ad_lengths.tile([A])
        else:
            cat_ad_lengths = (
                torch.cat(
                    [torch.tensor([L for _ in range(T * A)]) for _ in range(B)], 0
                )
                .cuda()
                .to(Dtype)
            )
            cat_ad_lengths_broadcasted = cat_ad_lengths
        batch_offsets = torch.tensor([A * b for b in range(B + 1)]).int().cuda()
        num_ads_in_batch = B * A
        reordered_batched_ad_lengths = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths, batch_offsets, num_ads_in_batch, broadcast_lengths
        )
        torch.testing.assert_close(
            cat_ad_lengths_broadcasted, reordered_batched_ad_lengths
        )

        cat_ad_lengths_cpu = cat_ad_lengths.cpu()
        batch_offsets_cpu = batch_offsets.cpu()
        reordered_batched_ad_lengths_cpu = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths_cpu, batch_offsets_cpu, num_ads_in_batch, broadcast_lengths
        )
        torch.testing.assert_close(
            reordered_batched_ad_lengths_cpu, reordered_batched_ad_lengths.cpu()
        )
//  [python]
def test_real(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_allclose(hidet.ops.real(a_hidet).torch(), torch.real(a))
//  [python]
def test_real(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_close(hidet.ops.real(a_hidet).torch(), torch.real(a))
//  [python]
def _run_test(
        self,
        model,
        inputs,
        device,
        dynamic=False,
        num_predicates=1,
    ):
        cnt = torch._dynamo.testing.CompileCounterWithBackend("inductor")
        compiled_model = torch.compile(backend=cnt, fullgraph=True)(model)

        inputs = [inp.to(device=device) for inp in inputs]
        input_sets = [inputs]
        if dynamic:
            larger_inputs = []
            for inp in inputs:
                # tile every first dim 5x
                tiling = [5] + [1] * (inp.ndim - 1)
                larger_inputs.append(torch.tile(inp, tiling))
            input_sets.append(larger_inputs)
            for inputs in input_sets:
                for inp in inputs:
                    # mark every first dim as dynamic
                    torch._dynamo.mark_dynamic(inp, 0)

        for inputs in input_sets:
            for inputs_with_predicates in prepend_predicates(inputs, num_predicates):
                cloned_inputs = [inp.clone() for inp in inputs_with_predicates]
                result = model(*inputs_with_predicates)
                result_compiled = compiled_model(*inputs_with_predicates)
                # inputs must not be mutated
                torch.testing.assert_allclose(cloned_inputs, inputs_with_predicates)
                torch.testing.assert_allclose(result, result_compiled)

        self.assertEqual(cnt.frame_count, 1, "only one compilation expected")
//  [python]
def _run_test(
        self,
        model,
        inputs,
        device,
        dynamic=False,
        num_predicates=1,
    ):
        cnt = torch._dynamo.testing.CompileCounterWithBackend("inductor")
        compiled_model = torch.compile(backend=cnt, fullgraph=True)(model)

        inputs = [inp.to(device=device) for inp in inputs]
        input_sets = [inputs]
        if dynamic:
            larger_inputs = []
            for inp in inputs:
                # tile every first dim 5x
                tiling = [5] + [1] * (inp.ndim - 1)
                larger_inputs.append(torch.tile(inp, tiling))
            input_sets.append(larger_inputs)
            for inputs in input_sets:
                for inp in inputs:
                    # mark every first dim as dynamic
                    torch._dynamo.mark_dynamic(inp, 0)

        for inputs in input_sets:
            for inputs_with_predicates in prepend_predicates(inputs, num_predicates):
                cloned_inputs = [inp.clone() for inp in inputs_with_predicates]
                result = model(*inputs_with_predicates)
                result_compiled = compiled_model(*inputs_with_predicates)
                # inputs must not be mutated
                torch.testing.assert_close(cloned_inputs, inputs_with_predicates)
                torch.testing.assert_close(result, result_compiled)

        self.assertEqual(cnt.frame_count, 1, "only one compilation expected")
//  [python]
def test_imag(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_allclose(hidet.ops.imag(a_hidet).torch(), torch.imag(a))
//  [python]
def test_imag(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_close(hidet.ops.imag(a_hidet).torch(), torch.imag(a))
//  [python]
def test_conj(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_allclose(hidet.ops.conj(a_hidet).torch(), torch.conj(a))
//  [python]
def test_conj(shape, dtype, device):
    a = torch.randn(shape, dtype=dtype, device=device)
    a_hidet = hidet.from_torch(a)

    torch.testing.assert_close(hidet.ops.conj(a_hidet).torch(), torch.conj(a))
//  [python]
def test_make_complex(shape, dtype, device):
    real = torch.randn(shape, dtype=dtype, device=device)
    imag = torch.randn(shape, dtype=dtype, device=device)
    a = torch.complex(real, imag)
    b = hidet.ops.make_complex(hidet.from_torch(real), hidet.from_torch(imag))

    torch.testing.assert_allclose(b.torch(), a)
//  [python]
def test_make_complex(shape, dtype, device):
    real = torch.randn(shape, dtype=dtype, device=device)
    imag = torch.randn(shape, dtype=dtype, device=device)
    a = torch.complex(real, imag)
    b = hidet.ops.make_complex(hidet.from_torch(real), hidet.from_torch(imag))

    torch.testing.assert_close(b.torch(), a)
//  [python]
def _run_test(
        self,
        model,
        inputs,
        device,
        dynamic=False,
        num_counters=1,
    ):
        cnt = torch._dynamo.testing.CompileCounterWithBackend("inductor")
        compiled_model = torch.compile(backend=cnt, fullgraph=True)(model)

        inputs = [inp.to(device=device) for inp in inputs]
        input_sets = [inputs]
        if dynamic:
            larger_inputs = []
            for inp in inputs:
                # tile every first dim 5x
                tiling = [5] + [1] * (inp.ndim - 1)
                larger_inputs.append(torch.tile(inp, tiling))
            input_sets.append(larger_inputs)
            for inputs in input_sets:
                for inp in inputs:
                    # mark every first dim as dynamic
                    if inp.ndim:
                        torch._dynamo.mark_dynamic(inp, 0)

        for inputs in input_sets:
            for inputs_with_counters in prepend_counters(inputs, num_counters):
                cloned_inputs = [inp.clone() for inp in inputs_with_counters]
                result = model(*inputs_with_counters)
                with torch.no_grad():
                    result_compiled = compiled_model(*inputs_with_counters)
                # inputs must not be mutated
                t.assert_close(cloned_inputs, inputs_with_counters)
                t.assert_close(
                    result, result_compiled, atol=1e-4, rtol=1e-4
                )

        self.assertEqual(cnt.frame_count, 1, "only one compilation expected")
//  [python]
def _run_test(
        self,
        model,
        inputs,
        device,
        dynamic=False,
        num_counters=1,
    ):
        cnt = torch._dynamo.testing.CompileCounterWithBackend("inductor")
        compiled_model = torch.compile(backend=cnt, fullgraph=True)(model)

        inputs = [inp.to(device=device) for inp in inputs]
        input_sets = [inputs]
        if dynamic:
            larger_inputs = []
            for inp in inputs:
                # tile every first dim 5x
                tiling = [5] + [1] * (inp.ndim - 1)
                larger_inputs.append(torch.tile(inp, tiling))
            input_sets.append(larger_inputs)
            for inputs in input_sets:
                for inp in inputs:
                    # mark every first dim as dynamic
                    if inp.ndim:
                        torch._dynamo.mark_dynamic(inp, 0)

        for inputs in input_sets:
            for inputs_with_counters in prepend_counters(inputs, num_counters):
                cloned_inputs = [inp.clone() for inp in inputs_with_counters]
                result = model(*inputs_with_counters)
                with torch.no_grad():
                    result_compiled = compiled_model(*inputs_with_counters)
                # inputs must not be mutated
                torch.testing.assert_close(cloned_inputs, inputs_with_counters)
                torch.testing.assert_close(
                    result, result_compiled, atol=1e-4, rtol=1e-4
                )

        self.assertEqual(cnt.frame_count, 1, "only one compilation expected")
//  [python]
def test_reorder_batched_ad_indices_cpu(
        self,
        B: int,
        T: int,
        L: int,
        A: int,
        Dtype: torch.dtype,
        Itype: torch.dtype,
        broadcast_indices: bool,
    ) -> None:
        if broadcast_indices:
            cat_ad_indices = (
                torch.randint(
                    low=0,
                    high=100,
                    size=(B * T * L,),
                )
                .int()
                .to(Dtype)
            )
            cat_ad_lengths = torch.cat(
                [torch.tensor([L for _ in range(T)]) for _ in range(B)],
                0,
            ).int()
            cat_ad_lengths_broadcasted = cat_ad_lengths.tile([A])
        else:
            cat_ad_indices = (
                torch.randint(
                    low=0,
                    high=100,
                    size=(B * T * A * L,),
                )
                .int()
                .to(Dtype)
            )
            cat_ad_lengths = torch.cat(
                [torch.tensor([L for _ in range(T * A)]) for _ in range(B)],
                0,
            ).int()
            cat_ad_lengths_broadcasted = cat_ad_lengths
        batch_offsets = torch.tensor([A * b for b in range(B + 1)]).int()
        num_ads_in_batch = B * A
        reordered_cat_ad_lengths = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths, batch_offsets, num_ads_in_batch, broadcast_indices
        )
        torch.testing.assert_allclose(cat_ad_lengths_broadcasted, reordered_cat_ad_lengths)
        cat_ad_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(
            cat_ad_lengths
        ).to(Itype)
        reordered_cat_ad_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(
            reordered_cat_ad_lengths
        ).to(Itype)
        reordered_cat_ad_indices = torch.ops.fbgemm.reorder_batched_ad_indices(
            cat_ad_offsets,
            cat_ad_indices,
            reordered_cat_ad_offsets,
            batch_offsets,
            num_ads_in_batch,
            broadcast_indices,
            B * T * A * L,
        )
        torch.testing.assert_allclose(
            reordered_cat_ad_indices.view(T, B, A, L).permute(1, 0, 2, 3),
            (
                cat_ad_indices.view(B, T, 1, L).tile([1, 1, A, 1])
                if broadcast_indices
                else cat_ad_indices.view(B, T, A, L)
            ),
        )
//  [python]
def test_reorder_batched_ad_indices_cpu(
        self,
        B: int,
        T: int,
        L: int,
        A: int,
        Dtype: torch.dtype,
        Itype: torch.dtype,
        broadcast_indices: bool,
    ) -> None:
        if broadcast_indices:
            cat_ad_indices = (
                torch.randint(
                    low=0,
                    high=100,
                    size=(B * T * L,),
                )
                .int()
                .to(Dtype)
            )
            cat_ad_lengths = torch.cat(
                [torch.tensor([L for _ in range(T)]) for _ in range(B)],
                0,
            ).int()
            cat_ad_lengths_broadcasted = cat_ad_lengths.tile([A])
        else:
            cat_ad_indices = (
                torch.randint(
                    low=0,
                    high=100,
                    size=(B * T * A * L,),
                )
                .int()
                .to(Dtype)
            )
            cat_ad_lengths = torch.cat(
                [torch.tensor([L for _ in range(T * A)]) for _ in range(B)],
                0,
            ).int()
            cat_ad_lengths_broadcasted = cat_ad_lengths
        batch_offsets = torch.tensor([A * b for b in range(B + 1)]).int()
        num_ads_in_batch = B * A
        reordered_cat_ad_lengths = torch.ops.fbgemm.reorder_batched_ad_lengths(
            cat_ad_lengths, batch_offsets, num_ads_in_batch, broadcast_indices
        )
        torch.testing.assert_close(cat_ad_lengths_broadcasted, reordered_cat_ad_lengths)
        cat_ad_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(
            cat_ad_lengths
        ).to(Itype)
        reordered_cat_ad_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(
            reordered_cat_ad_lengths
        ).to(Itype)
        reordered_cat_ad_indices = torch.ops.fbgemm.reorder_batched_ad_indices(
            cat_ad_offsets,
            cat_ad_indices,
            reordered_cat_ad_offsets,
            batch_offsets,
            num_ads_in_batch,
            broadcast_indices,
            B * T * A * L,
        )
        torch.testing.assert_close(
            reordered_cat_ad_indices.view(T, B, A, L).permute(1, 0, 2, 3),
            (
                cat_ad_indices.view(B, T, 1, L).tile([1, 1, A, 1])
                if broadcast_indices
                else cat_ad_indices.view(B, T, A, L)
            ),
        )
//  [python]
def test_gather(self, input, dim, num):
        model = _TestModel(num, dim).eval().cuda()
        dummy_input = torch.zeros_like(input)
        trt_model = module2trt(model, args=[dummy_input])

        with torch.inference_mode():
            gt = model(input)
            out = trt_model(input)
        torch.testing.assert_allclose(out, gt)
//  [python]
def test_gather(self, input, dim, num):
        model = _TestModel(num, dim).eval().cuda()
        dummy_input = torch.zeros_like(input)
        trt_model = module2trt(model, args=[dummy_input])

        with torch.inference_mode():
            gt = model(input)
            out = trt_model(input)
        torch.testing.assert_close(out, gt)
//  [python]
def test_complex_matmul(a_shape, b_shape, dtype, device):
    a = torch.randn(a_shape, dtype=dtype, device=device)
    b = torch.randn(b_shape, dtype=dtype, device=device)
    c = torch.matmul(a, b)

    a_hidet = hidet.from_torch(a)
    b_hidet = hidet.from_torch(b)
    c_hidet = hidet.ops.matmul(a_hidet, b_hidet)

    torch.testing.assert_allclose(c_hidet.torch(), c, atol=1e-5, rtol=1e-5)
//  [python]
def test_complex_matmul(a_shape, b_shape, dtype, device):
    a = torch.randn(a_shape, dtype=dtype, device=device)
    b = torch.randn(b_shape, dtype=dtype, device=device)
    c = torch.matmul(a, b)

    a_hidet = hidet.from_torch(a)
    b_hidet = hidet.from_torch(b)
    c_hidet = hidet.ops.matmul(a_hidet, b_hidet)

    torch.testing.assert_close(c_hidet.torch(), c, atol=1e-5, rtol=1e-5)
//  [python]
def test_float_round_trip(self, dtype: torch.dtype):
    t = torch.randn((3, 3), dtype=dtype)
    xt = t.to(xm.xla_device())
    torch.testing.assert_allclose(xt.cpu(), t)
//  [python]
def test_float_round_trip(self, dtype: torch.dtype):
    t = torch.randn((3, 3), dtype=dtype)
    xt = t.to(xm.xla_device())
    torch.testing.assert_close(xt.cpu(), t)
//  [python]
def test_int_round_trip(self, dtype: torch.dtype):
    t = torch.randint(0, 128, (3, 3), dtype=dtype)
    xt = t.to(xm.xla_device())
    torch.testing.assert_allclose(xt.cpu(), t)
//  [python]
def test_int_round_trip(self, dtype: torch.dtype):
    t = torch.randint(0, 128, (3, 3), dtype=dtype)
    xt = t.to(xm.xla_device())
    torch.testing.assert_close(xt.cpu(), t)
//  [python]
def test_bool_round_trip(self):
    t = torch.randint(0, 2, (3, 3), dtype=torch.bool)
    xt = t.to(xm.xla_device())
    torch.testing.assert_allclose(xt.cpu(), t)
//  [python]
def test_bool_round_trip(self):
    t = torch.randint(0, 2, (3, 3), dtype=torch.bool)
    xt = t.to(xm.xla_device())
    torch.testing.assert_close(xt.cpu(), t)
//  [python]
def test_compare_binding_to_pytorch():
    from gsplat._torch_impl import compute_cov2d_bounds as _compute_cov2d_bounds
    from gsplat import compute_cov2d_bounds

    torch.manual_seed(42)

    num_cov2ds = 100

    _covs2d = torch.rand(
        (num_cov2ds, 2, 2), dtype=torch.float32, device=device, requires_grad=True
    )
    covs2d = torch.stack(
        [
            torch.triu(_covs2d)[:, 0, 0],
            torch.triu(_covs2d)[:, 0, 1],
            torch.triu(_covs2d)[:, 1, 1],
        ],
        dim=-1,
    )

    conic, radii = compute_cov2d_bounds(covs2d)
    _conic, _radii, _mask = _compute_cov2d_bounds(_covs2d)

    radii = radii.squeeze(-1)

    atol = 5e-4
    rtol = 1e-5
    torch.allclose(conic[_mask], _conic[_mask], atol=atol, rtol=rtol)
    torch.allclose(radii[_mask], _radii[_mask], atol=atol, rtol=rtol)
//  [python]
def test_compare_binding_to_pytorch():
    from gsplat._torch_impl import compute_cov2d_bounds as _compute_cov2d_bounds
    from gsplat import compute_cov2d_bounds

    torch.manual_seed(42)

    num_cov2ds = 100

    _covs2d = torch.rand(
        (num_cov2ds, 2, 2), dtype=torch.float32, device=device, requires_grad=True
    )
    covs2d = torch.stack(
        [
            torch.triu(_covs2d)[:, 0, 0],
            torch.triu(_covs2d)[:, 0, 1],
            torch.triu(_covs2d)[:, 1, 1],
        ],
        dim=-1,
    )

    conic, radii = compute_cov2d_bounds(covs2d)
    _conic, _radii, _mask = _compute_cov2d_bounds(_covs2d)

    radii = radii.squeeze(-1)

    atol = 5e-4
    rtol = 1e-5
    torch.testing.assert_close(conic[_mask], _conic[_mask], atol=atol, rtol=rtol)
    torch.testing.assert_close(radii[_mask], _radii[_mask], atol=atol, rtol=rtol)
//  [python]
def test_cdf(x, q):
    # Pseudo-inverse property
    p = cdf(x, q)

    # Strictly speaking, icdf ought to return -inf when p = 0, but in practice we
    # want to clip to the minimum value, so we've implemented it that way.
    p_inv = icdf(p, q).where(p > 0, -torch.inf)
    assert torch.all(p_inv <= x)
    assert torch.all(cdf(p_inv, q) >= p)

    # Should match SciPy
    t.assert_close(
        cdf(x, q),
        torch.from_numpy(scipy_cdf(q).cdf.evaluate(x)).float(),
    )
//  [python]
def test_cdf(x, q):
    # Pseudo-inverse property
    p = cdf(x, q)

    # Strictly speaking, icdf ought to return -inf when p = 0, but in practice we
    # want to clip to the minimum value, so we've implemented it that way.
    p_inv = icdf(p, q).where(p > 0, -torch.inf)
    assert torch.all(p_inv <= x)
    assert torch.all(cdf(p_inv, q) >= p)

    # Should match SciPy
    torch.testing.assert_close(
        cdf(x, q),
        torch.from_numpy(scipy_cdf(q).cdf.evaluate(x)).float(),
    )
//  [python]
def test_recat_embedding_grad_output(self, B: int, T: int, D: int, W: int) -> None:
        num_features_per_rank = np.random.randint(low=1, high=20, size=(W,)).tolist()
        grad_output = torch.randn(B, sum(num_features_per_rank), D).float().cuda()
        grad_outputs_by_rank = grad_output.split(num_features_per_rank, dim=1)
        sharded_grad_output = torch.cat(
            [
                grad_output_by_rank.contiguous().view(-1)
                for grad_output_by_rank in grad_outputs_by_rank
            ],
            dim=0,
        )
        sharded_grad_output_impl = torch.ops.fbgemm.recat_embedding_grad_output(
            grad_output, num_features_per_rank
        )
        torch.testing.assert_allclose(
            sharded_grad_output_impl.cpu(), sharded_grad_output.cpu()
        )
//  [python]
def test_recat_embedding_grad_output(self, B: int, T: int, D: int, W: int) -> None:
        num_features_per_rank = np.random.randint(low=1, high=20, size=(W,)).tolist()
        grad_output = torch.randn(B, sum(num_features_per_rank), D).float().cuda()
        grad_outputs_by_rank = grad_output.split(num_features_per_rank, dim=1)
        sharded_grad_output = torch.cat(
            [
                grad_output_by_rank.contiguous().view(-1)
                for grad_output_by_rank in grad_outputs_by_rank
            ],
            dim=0,
        )
        sharded_grad_output_impl = torch.ops.fbgemm.recat_embedding_grad_output(
            grad_output, num_features_per_rank
        )
        torch.testing.assert_close(
            sharded_grad_output_impl.cpu(), sharded_grad_output.cpu()
        )
//  [python]
def _test_fused_dense(self, dtype, seed=0):

        os.environ["TORCH_ALLOW_TF32_CUBLAS_OVERRIDE"] = "0"
        torch.manual_seed(seed)

        seq_length = 512
        sequences = 3
        hidden_dim = 1024

        ref_inputs = torch.randn(sequences*seq_length, hidden_dim,
                                 dtype=dtype, device=torch.device("cuda")).requires_grad_(True)

        tst_inputs = ref_inputs.clone().detach().requires_grad_(True)
        dense = fused_dense.FusedDense(1024, 3072)
        dense.to(dtype=dtype)
        dense.cuda()

        y_tst = dense(tst_inputs)
        y_ref = torch.matmul(ref_inputs, dense.weight.t())+dense.bias
        dy = torch.randn_like(y_tst).to(dtype=dtype)
        y_tst.backward(dy)
        dw_ref = torch.matmul(dy.t(), ref_inputs)
        dx_ref = torch.matmul(dy, dense.weight.clone())
        db_ref = dy.sum(0, False)

        torch.testing.assert_allclose(
            ref_inputs,  tst_inputs,  atol=1e-5, rtol=1e-5)
        torch.testing.assert_allclose(
            y_ref,  y_tst,  atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_allclose(
            dw_ref, dense.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_allclose(
            dx_ref, tst_inputs.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_allclose(
            db_ref, dense.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
//  [python]
def _test_fused_dense(self, dtype, seed=0):

        os.environ["TORCH_ALLOW_TF32_CUBLAS_OVERRIDE"] = "0"
        torch.manual_seed(seed)

        seq_length = 512
        sequences = 3
        hidden_dim = 1024

        ref_inputs = torch.randn(sequences*seq_length, hidden_dim,
                                 dtype=dtype, device=torch.device("cuda")).requires_grad_(True)

        tst_inputs = ref_inputs.clone().detach().requires_grad_(True)
        dense = fused_dense.FusedDense(1024, 3072)
        dense.to(dtype=dtype)
        dense.cuda()

        y_tst = dense(tst_inputs)
        y_ref = torch.matmul(ref_inputs, dense.weight.t())+dense.bias
        dy = torch.randn_like(y_tst).to(dtype=dtype)
        y_tst.backward(dy)
        dw_ref = torch.matmul(dy.t(), ref_inputs)
        dx_ref = torch.matmul(dy, dense.weight.clone())
        db_ref = dy.sum(0, False)

        torch.testing.assert_close(
            ref_inputs,  tst_inputs,  atol=1e-5, rtol=1e-5)
        torch.testing.assert_close(
            y_ref,  y_tst,  atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_close(
            dw_ref, dense.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_close(
            dx_ref, tst_inputs.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
        torch.testing.assert_close(
            db_ref, dense.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True)
//  [python]
def _test_binary_recall_with_input(
        self,
        input: torch.Tensor,
        target: torch.Tensor,
        threshold: float = 0.5,
    ) -> None:

        my_compute_result = my_binary_recall_score(input, target, threshold=threshold)

        input_np = np.where(input.numpy() < threshold, 0, 1)
        target_np = target.squeeze().numpy()

        sklearn_result = torch.tensor(
            ref_recall_score(target_np, input_np, average="binary")
        ).to(torch.float32)

        torch.testing.assert_allclose(
            my_compute_result,
            sklearn_result,
            equal_nan=True,
            atol=1e-8,
            rtol=1e-5,
        )
//  [python]
def _test_binary_recall_with_input(
        self,
        input: torch.Tensor,
        target: torch.Tensor,
        threshold: float = 0.5,
    ) -> None:

        my_compute_result = my_binary_recall_score(input, target, threshold=threshold)

        input_np = np.where(input.numpy() < threshold, 0, 1)
        target_np = target.squeeze().numpy()

        sklearn_result = torch.tensor(
            ref_recall_score(target_np, input_np, average="binary")
        ).to(torch.float32)

        torch.testing.assert_close(
            my_compute_result,
            sklearn_result,
            equal_nan=True,
            atol=1e-8,
            rtol=1e-5,
        )
//  [python]
def test_aligned_iou(dims1, dims2, expected_ious, catch_warnings):
    warnings.filterwarnings(
        "ignore",
        message=".*does not have many workers which may be a bottleneck.*",
        category=PossibleUserWarning,
    )

    t.assert_close(aligned_iou(dims1, dims2), expected_ious)
//  [python]
def test_aligned_iou(dims1, dims2, expected_ious, catch_warnings):
    warnings.filterwarnings(
        "ignore",
        message=".*does not have many workers which may be a bottleneck.*",
        category=PossibleUserWarning,
    )

    torch.testing.assert_close(aligned_iou(dims1, dims2), expected_ious)
//  [python]
def test_recat_embedding_grad_output_mixed_D(
        self, B: int, W: int, cuda: bool
    ) -> None:
        num_features_per_rank = np.random.randint(low=1, high=20, size=(W,)).tolist()
        global_T = sum(num_features_per_rank)
        mixed_D_list = np.random.randint(low=1, high=10, size=(global_T,))
        grad_output = torch.randn(B, sum(mixed_D_list)).float().cuda()
        if cuda:
            grad_output = grad_output.cuda()
        num_feature_offsets_list = torch.tensor(
            [0] + np.cumsum(num_features_per_rank).tolist()
        )
        dim_sum_per_rank = [
            sum(
                mixed_D_list[
                    num_feature_offsets_list[i] : num_feature_offsets_list[i + 1]
                ]
            )
            for i in range(W)
        ]
        grad_outputs_by_rank = grad_output.split(dim_sum_per_rank, dim=1)
        sharded_grad_output = torch.cat(
            [
                grad_output_by_rank.contiguous().view(-1)
                for grad_output_by_rank in grad_outputs_by_rank
            ],
            dim=0,
        )
        sharded_grad_output_impl = torch.ops.fbgemm.recat_embedding_grad_output_mixed_D(
            grad_output, dim_sum_per_rank
        )
        torch.testing.assert_allclose(
            sharded_grad_output_impl.cpu(), sharded_grad_output.cpu()
        )
//  [python]
def test_recat_embedding_grad_output_mixed_D(
        self, B: int, W: int, cuda: bool
    ) -> None:
        num_features_per_rank = np.random.randint(low=1, high=20, size=(W,)).tolist()
        global_T = sum(num_features_per_rank)
        mixed_D_list = np.random.randint(low=1, high=10, size=(global_T,))
        grad_output = torch.randn(B, sum(mixed_D_list)).float().cuda()
        if cuda:
            grad_output = grad_output.cuda()
        num_feature_offsets_list = torch.tensor(
            [0] + np.cumsum(num_features_per_rank).tolist()
        )
        dim_sum_per_rank = [
            sum(
                mixed_D_list[
                    num_feature_offsets_list[i] : num_feature_offsets_list[i + 1]
                ]
            )
            for i in range(W)
        ]
        grad_outputs_by_rank = grad_output.split(dim_sum_per_rank, dim=1)
        sharded_grad_output = torch.cat(
            [
                grad_output_by_rank.contiguous().view(-1)
                for grad_output_by_rank in grad_outputs_by_rank
            ],
            dim=0,
        )
        sharded_grad_output_impl = torch.ops.fbgemm.recat_embedding_grad_output_mixed_D(
            grad_output, dim_sum_per_rank
        )
        torch.testing.assert_close(
            sharded_grad_output_impl.cpu(), sharded_grad_output.cpu()
        )
//  [python]
def _test_recall_with_input(
        self,
        input: torch.Tensor,
        target: torch.Tensor,
        num_classes: Optional[int] = None,
        average: Optional[str] = "micro",
    ) -> None:
        if num_classes is None:
            if average == "micro":
                my_compute_result = my_recall_score(input, target)
            else:
                my_compute_result = my_recall_score(input, target, average=average)
        else:
            if average == "micro":
                my_compute_result = my_recall_score(
                    input, target, num_classes=num_classes
                )
            else:
                my_compute_result = my_recall_score(
                    input, target, num_classes=num_classes, average=average
                )
        target = target.squeeze().flatten()
        input_squeezed = input.squeeze()
        input_label_ids = (
            torch.argmax(input_squeezed, dim=1)
            if input_squeezed.ndim == 2
            else input_squeezed
        )
        input = input_label_ids.flatten()
        compute_result = torch.tensor(
            ref_recall_score(target, input, average=average)
        ).to(torch.float32)
        if my_compute_result.shape != compute_result.shape:
            compute_result = torch.from_numpy(
                np.append(compute_result.numpy(), 0.0)
            ).to(torch.float32)
        torch.testing.assert_allclose(
            my_compute_result,
            compute_result,
            equal_nan=True,
            atol=1e-8,
            rtol=1e-5,
        )
//  [python]
def _test_recall_with_input(
        self,
        input: torch.Tensor,
        target: torch.Tensor,
        num_classes: Optional[int] = None,
        average: Optional[str] = "micro",
    ) -> None:
        if num_classes is None:
            if average == "micro":
                my_compute_result = my_recall_score(input, target)
            else:
                my_compute_result = my_recall_score(input, target, average=average)
        else:
            if average == "micro":
                my_compute_result = my_recall_score(
                    input, target, num_classes=num_classes
                )
            else:
                my_compute_result = my_recall_score(
                    input, target, num_classes=num_classes, average=average
                )
        target = target.squeeze().flatten()
        input_squeezed = input.squeeze()
        input_label_ids = (
            torch.argmax(input_squeezed, dim=1)
            if input_squeezed.ndim == 2
            else input_squeezed
        )
        input = input_label_ids.flatten()
        compute_result = torch.tensor(
            ref_recall_score(target, input, average=average)
        ).to(torch.float32)
        if my_compute_result.shape != compute_result.shape:
            compute_result = torch.from_numpy(
                np.append(compute_result.numpy(), 0.0)
            ).to(torch.float32)
        torch.testing.assert_close(
            my_compute_result,
            compute_result,
            equal_nan=True,
            atol=1e-8,
            rtol=1e-5,
        )
//  [python]
def test_flash_attn_unit():
    model = FlashAttnModel().to('cuda')
    model_golden = FlashAttnModel().to('cuda')
    model_golden.load_state_dict(model.state_dict())

    q = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    k = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    v = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    q_clone = q.clone().detach().requires_grad_()
    k_clone = k.clone().detach().requires_grad_()
    v_clone = v.clone().detach().requires_grad_()
    model.zero_grad(set_to_none=True)
    model_golden.zero_grad(set_to_none=True)
    optimized_model = torch.compile(model, backend=byteir_compile_fx)
    output = optimized_model(q, k, v)
    golden_output = model_golden(q_clone, k_clone, v_clone)
    torch.testing.assert_allclose(output, golden_output)
    output.mean().backward()
    golden_output.mean().backward()
    torch.testing.assert_allclose(q.grad, q_clone.grad)
    torch.testing.assert_allclose(k.grad, k_clone.grad)
    torch.testing.assert_allclose(v.grad, v_clone.grad)
//  [python]
def test_flash_attn_unit():
    model = FlashAttnModel().to('cuda')
    model_golden = FlashAttnModel().to('cuda')
    model_golden.load_state_dict(model.state_dict())

    q = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    k = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    v = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    q_clone = q.clone().detach().requires_grad_()
    k_clone = k.clone().detach().requires_grad_()
    v_clone = v.clone().detach().requires_grad_()
    model.zero_grad(set_to_none=True)
    model_golden.zero_grad(set_to_none=True)
    optimized_model = torch.compile(model, backend=byteir_compile_fx)
    output = optimized_model(q, k, v)
    golden_output = model_golden(q_clone, k_clone, v_clone)
    torch.testing.assert_close(output, golden_output)
    output.mean().backward()
    golden_output.mean().backward()
    torch.testing.assert_close(q.grad, q_clone.grad)
    torch.testing.assert_close(k.grad, k_clone.grad)
    torch.testing.assert_close(v.grad, v_clone.grad)
//  [python]
def test_not_looking_at_the_future(self):
        test_len = 20
        x = torch.rand(5, test_len, 8)
        batch_lens = torch.LongTensor([test_len] * 5)
        padding_mask = lengths_to_padding_mask(batch_lens)
        encoder = HyenaOperator(8, test_len, num_heads=4)
        output = encoder.forward(x, padding_mask)
        for j in range(19):
            # Checks that for each of the 20 elements we obtain the same prefix in the
            # results when feeding the model with the full input sequences and the input
            # prefix truncated at that element.
            partial_lens = torch.LongTensor([j + 1] * 5)
            partial_padding_mask = lengths_to_padding_mask(partial_lens)
            partial_output = encoder.forward(x[:, :j + 1, :], partial_padding_mask)
            torch.testing.assert_allclose(
                partial_output,
                output[:, :j + 1, :])
//  [python]
def test_not_looking_at_the_future(self):
        test_len = 20
        x = torch.rand(5, test_len, 8)
        batch_lens = torch.LongTensor([test_len] * 5)
        padding_mask = lengths_to_padding_mask(batch_lens)
        encoder = HyenaOperator(8, test_len, num_heads=4)
        output = encoder.forward(x, padding_mask)
        for j in range(19):
            # Checks that for each of the 20 elements we obtain the same prefix in the
            # results when feeding the model with the full input sequences and the input
            # prefix truncated at that element.
            partial_lens = torch.LongTensor([j + 1] * 5)
            partial_padding_mask = lengths_to_padding_mask(partial_lens)
            partial_output = encoder.forward(x[:, :j + 1, :], partial_padding_mask)
            torch.testing.assert_close(
                partial_output,
                output[:, :j + 1, :])
//  [python]
def test_noncausal(self):
        test_len = 20
        x = torch.rand(5, test_len, 8)
        batch_lens = torch.LongTensor([test_len] * 5)
        padding_mask = lengths_to_padding_mask(batch_lens)
        encoder = HyenaOperator(8, test_len, num_heads=4, causal=False)
        output = encoder.forward(x, padding_mask)
        for j in range(19):
            # Checks that for each of the 20 elements we obtain the same prefix in the
            # results when feeding the model with the full input sequences and the input
            # prefix truncated at that element.
            partial_lens = torch.LongTensor([j + 1] * 5)
            partial_padding_mask = lengths_to_padding_mask(partial_lens)
            partial_output = encoder.forward(x[:, :j + 1, :], partial_padding_mask)
            self.assertTrue(torch.allclose(partial_output, output[:, :j + 1, :]))
//  [python]
def test_noncausal(self):
        test_len = 20
        x = torch.rand(5, test_len, 8)
        batch_lens = torch.LongTensor([test_len] * 5)
        padding_mask = lengths_to_padding_mask(batch_lens)
        encoder = HyenaOperator(8, test_len, num_heads=4, causal=False)
        output = encoder.forward(x, padding_mask)
        for j in range(19):
            # Checks that for each of the 20 elements we obtain the same prefix in the
            # results when feeding the model with the full input sequences and the input
            # prefix truncated at that element.
            partial_lens = torch.LongTensor([j + 1] * 5)
            partial_padding_mask = lengths_to_padding_mask(partial_lens)
            partial_output = encoder.forward(x[:, :j + 1, :], partial_padding_mask)
            with self.assertRaises(AssertionError):
                torch.testing.assert_close(
                    partial_output,
                    output[:, :j + 1, :])
//  [python]
def test_flash_attn_functional_unit():
    model = FlashAttnFunctionalModel().to('cuda')
    model_golden = FlashAttnFunctionalModel().to('cuda')
    model_golden.load_state_dict(model.state_dict())

    q = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    k = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    v = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    q_clone = q.clone().detach().requires_grad_()
    k_clone = k.clone().detach().requires_grad_()
    v_clone = v.clone().detach().requires_grad_()
    model.zero_grad(set_to_none=True)
    model_golden.zero_grad(set_to_none=True)
    optimized_model = torch.compile(model, backend=byteir_compile_fx)
    output = optimized_model(q, k, v)
    golden_output = model_golden(q_clone, k_clone, v_clone)
    torch.allclose(output, golden_output)
    output.mean().backward()
    golden_output.mean().backward()
    torch.allclose(q.grad, q_clone.grad)
    torch.allclose(k.grad, k_clone.grad)
    torch.allclose(v.grad, v_clone.grad)
//  [python]
def test_flash_attn_functional_unit():
    model = FlashAttnFunctionalModel().to('cuda')
    model_golden = FlashAttnFunctionalModel().to('cuda')
    model_golden.load_state_dict(model.state_dict())

    q = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    k = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    v = torch.rand(2, 12, 256, 128, dtype=torch.half, device="cuda:0", requires_grad=True)
    q_clone = q.clone().detach().requires_grad_()
    k_clone = k.clone().detach().requires_grad_()
    v_clone = v.clone().detach().requires_grad_()
    model.zero_grad(set_to_none=True)
    model_golden.zero_grad(set_to_none=True)
    optimized_model = torch.compile(model, backend=byteir_compile_fx)
    output = optimized_model(q, k, v)
    golden_output = model_golden(q_clone, k_clone, v_clone)
    torch.testing.assert_close(output, golden_output)
    output.mean().backward()
    golden_output.mean().backward()
    torch.testing.assert_close(q.grad, q_clone.grad)
    torch.testing.assert_close(k.grad, k_clone.grad)
    torch.testing.assert_close(v.grad, v_clone.grad)
//  [python]
def test_cdf(self):
        t.assert_close(
            UniformNoise(self.distribution).cdf(self.x),
            self.distribution.cdf(self.x),
        )
//  [python]
def test_cdf(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).cdf(self.x),
            self.distribution.cdf(self.x),
        )
//  [python]
def test_entropy(self):
        self.assertEqual(
            UniformNoise(self.distribution).entropy(),
            self.distribution.entropy(),
        )
//  [python]
def test_entropy(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).entropy(),
            self.distribution.entropy(),
        )
//  [python]
def test_expand(self):
        self.assertEqual(
            UniformNoise(self.distribution).expand(Size([32])).batch_shape,
            Size([32]),
        )
//  [python]
def test_expand(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).expand(Size([32])).batch_shape,
            Size([32]),
        )
//  [python]
def test_icdf(self):
        self.assertEqual(
            UniformNoise(self.distribution).icdf(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_icdf(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).icdf(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_log_prob(self):
        t.assert_close(
            UniformNoise(self.distribution).log_prob(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_log_prob(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).log_prob(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_log_cdf(self):
        t.assert_close(
            UniformNoise(self.distribution).log_cdf(self.x),
            ncF.log_cdf(self.x, self.distribution),
        )
//  [python]
def test_log_cdf(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).log_cdf(self.x),
            ncF.log_cdf(self.x, self.distribution),
        )
//  [python]
def test_optimizer_lr_one_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1 = 0.1

    named_params_or_groups = []
    for name, param in model.named_parameters():
        named_params_or_groups.append((name, param))
    named_params_or_groups = [{"named_params": named_params_or_groups, "lr": lr1}]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        fc1_grad = model.fc1.weight.grad.clone()
        fc2_grad = model.fc2.weight.grad.clone()

        # compute gradient manually
        with torch.no_grad():
            expected_fc1_weight = model.fc1.weight - lr1 * fc1_grad
            expected_fc2_weight = model.fc2.weight - lr1 * fc2_grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        t.assert_close(expected_fc1_weight, updated_fc1_weight)
        t.assert_close(expected_fc2_weight, updated_fc2_weight)