jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_flash_attn_llama_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.LlamaConfig(num_hidden_layers=2)
    config.hidden_size=512
    model = transformers.LlamaForCausalLM(config=config).to("cuda")

    flash_model = transformers.LlamaForCausalLM(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    model.zero_grad(set_to_none=True)
    flash_model.zero_grad(set_to_none=True)
    flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    # Flash attention uses a different mask than the original llama, relax check
    torch.testing.assert_close(golden_loss, flash_loss, atol=3e-4, rtol=1e-6)
    torch.testing.assert_close(golden_logits, flash_logits, atol=3e-2, rtol=1e-6)
//  [python]
def test_flash_attn_gpt2_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.GPT2Config.from_pretrained('gpt2')
    config.num_labels = config.vocab_size
    config.attn_pdrop = 0.0
    config.resid_pdrop = 0.0
    config.embd_pdrop = 0.0
    config.classifier_dropout = 0.0
    model = transformers.GPT2ForTokenClassification(config=config).to("cuda")

    flash_model = transformers.GPT2ForTokenClassification(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    model.zero_grad(set_to_none=True)
    flash_model.zero_grad(set_to_none=True)
    flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    torch.testing.assert_allclose(golden_loss, flash_loss, atol=3e-4, rtol=1e-6)
    torch.testing.assert_allclose(golden_logits, flash_logits, atol=4e-3, rtol=1e-5)
//  [python]
def test_flash_attn_gpt2_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.GPT2Config.from_pretrained('gpt2')
    config.num_labels = config.vocab_size
    config.attn_pdrop = 0.0
    config.resid_pdrop = 0.0
    config.embd_pdrop = 0.0
    config.classifier_dropout = 0.0
    model = transformers.GPT2ForTokenClassification(config=config).to("cuda")

    flash_model = transformers.GPT2ForTokenClassification(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    model.zero_grad(set_to_none=True)
    flash_model.zero_grad(set_to_none=True)
    flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    torch.testing.assert_close(golden_loss, flash_loss, atol=3e-4, rtol=1e-6)
    torch.testing.assert_close(golden_logits, flash_logits, atol=4e-3, rtol=1e-5)
//  [python]
def test_flash_attn_bloom_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.BloomConfig.from_pretrained('bigscience/bloom-560m')
    config.tie_word_embeddings = False
    config.hidden_size=512
    config.num_hidden_layers=2
    model = transformers.BloomForCausalLM(config=config).to("cuda")

    flash_model = transformers.BloomForCausalLM(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        model.zero_grad(set_to_none=True)
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_model.zero_grad(set_to_none=True)
        flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    # Bloom uses Alibi tensor, which is not causal. Transformation is not mathematically equivalent
    torch.testing.assert_allclose(golden_loss, flash_loss, atol=1e-3, rtol=1e-6)
//  [python]
def test_flash_attn_bloom_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.BloomConfig.from_pretrained('bigscience/bloom-560m')
    config.tie_word_embeddings = False
    config.hidden_size=512
    config.num_hidden_layers=2
    model = transformers.BloomForCausalLM(config=config).to("cuda")

    flash_model = transformers.BloomForCausalLM(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        model.zero_grad(set_to_none=True)
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_model.zero_grad(set_to_none=True)
        flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    # Bloom uses Alibi tensor, which is not causal. Transformation is not mathematically equivalent
    torch.testing.assert_close(golden_loss, flash_loss, atol=1e-3, rtol=1e-6)
//  [python]
def test_mm(self):
        B = torch.ones(5, 6).to("mps")
        C = torch.ones(6, 5).to("mps")
        D = torch.mm(B, C).cpu()
        torch.testing.assert_allclose(D, torch.full((5, 5), 6.0))
//  [python]
def test_mm(self):
        B = torch.ones(5, 6).to("mps")
        C = torch.ones(6, 5).to("mps")
        D = torch.mm(B, C).cpu()
        torch.testing.assert_close(D, torch.full((5, 5), 6.0))
//  [python]
def _testLeakyRelu(self, shape, dtype, negative_slope, contiguous):
        cpu_x = torch.randn(shape, device='cpu', dtype=dtype)
        mps_x = cpu_x.detach().clone().to('mps')

        if not contiguous and not (0 in shape or len(shape) < 2):
            # Tranposing will make the tensor non-contiguous
            cpu_x = cpu_x.transpose(0, 1)
            mps_x = mps_x.transpose(0, 1)
            assert not mps_x.is_contiguous()

        cpu_x.requires_grad_()
        mps_x.requires_grad_()

        relu_op = torch.nn.LeakyReLU(negative_slope)

        cpu_leaky_relu = relu_op(cpu_x)
        mps_leaky_relu = relu_op(mps_x)
        torch.testing.assert_allclose(cpu_leaky_relu, mps_leaky_relu.to('cpu'))

        # test backward pass

        cpu_grad = torch.ones_like(cpu_leaky_relu)
        mps_grad = cpu_grad.to('mps')

        mps_leaky_relu.backward(gradient=mps_grad)
        cpu_leaky_relu.backward(gradient=cpu_grad)

        assert cpu_x.grad is not None  # Check that the grad is well-populated
        self.assertEqual(cpu_x.grad, mps_x.grad)
//  [python]
def _testLeakyRelu(self, shape, dtype, negative_slope, contiguous):
        cpu_x = torch.randn(shape, device='cpu', dtype=dtype)
        mps_x = cpu_x.detach().clone().to('mps')

        if not contiguous and not (0 in shape or len(shape) < 2):
            # Tranposing will make the tensor non-contiguous
            cpu_x = cpu_x.transpose(0, 1)
            mps_x = mps_x.transpose(0, 1)
            assert not mps_x.is_contiguous()

        cpu_x.requires_grad_()
        mps_x.requires_grad_()

        relu_op = torch.nn.LeakyReLU(negative_slope)

        cpu_leaky_relu = relu_op(cpu_x)
        mps_leaky_relu = relu_op(mps_x)
        torch.testing.assert_close(cpu_leaky_relu, mps_leaky_relu.to('cpu'))

        # test backward pass

        cpu_grad = torch.ones_like(cpu_leaky_relu)
        mps_grad = cpu_grad.to('mps')

        mps_leaky_relu.backward(gradient=mps_grad)
        cpu_leaky_relu.backward(gradient=cpu_grad)

        assert cpu_x.grad is not None  # Check that the grad is well-populated
        self.assertEqual(cpu_x.grad, mps_x.grad)
//  [python]
def test_flash_attn_opt_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.AutoConfig.from_pretrained("facebook/opt-1.3b")
    config.tie_word_embeddings = False
    config.hidden_size=512
    config.num_hidden_layers=2
    config.dropout=0.0
    model = transformers.OPTForCausalLM(config=config).to("cuda")

    flash_model = transformers.OPTForCausalLM(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        model.zero_grad(set_to_none=True)
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_model.zero_grad(set_to_none=True)
        flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    torch.testing.assert_allclose(golden_loss, flash_loss, atol=2e-4, rtol=2e-6)
    torch.testing.assert_allclose(golden_logits, flash_logits, atol=3e-3, rtol=1e-6)
//  [python]
def test_flash_attn_opt_pattern():
    torch.cuda.empty_cache()
    torch.manual_seed(0)
    config = transformers.AutoConfig.from_pretrained("facebook/opt-1.3b")
    config.tie_word_embeddings = False
    config.hidden_size=512
    config.num_hidden_layers=2
    config.dropout=0.0
    model = transformers.OPTForCausalLM(config=config).to("cuda")

    flash_model = transformers.OPTForCausalLM(config=config).to("cuda")
    flash_model.load_state_dict(model.state_dict())

    data = make_data(model, "cuda")
    flash_data = copy_nested_tensor(data)

    with torch.cuda.amp.autocast(enabled=True, dtype=torch.float16):
        model.zero_grad(set_to_none=True)
        input_data, label = data
        output = flash_model(input_data)
        golden_logits = output.logits
        golden_loss = F.cross_entropy(golden_logits.view(-1, model.config.vocab_size), label.view(-1))

        flash_model.zero_grad(set_to_none=True)
        flash_attn_gm = torch.compile(flash_model, backend=trival_compile_fx)
        flash_input_data, flash_label = flash_data
        flash_output = flash_attn_gm(flash_input_data)
        flash_logits = flash_output["logits"]
        flash_loss = F.cross_entropy(flash_logits.view(-1, model.config.vocab_size), flash_label.view(-1))

    torch.testing.assert_close(golden_loss, flash_loss, atol=2e-4, rtol=2e-6)
    torch.testing.assert_close(golden_logits, flash_logits, atol=3e-3, rtol=1e-6)
//  [python]
def _testLeakyRelu(self, np_features, negative_slope, device):
        cpu_x = torch.from_numpy(np_features).requires_grad_()
        mps_x = torch.from_numpy(np_features).to('mps').requires_grad_()
        relu_op = torch.nn.LeakyReLU(negative_slope)

        cpu_leaky_relu = relu_op(cpu_x)
        mps_leaky_relu = relu_op(mps_x)
        torch.testing.assert_allclose(cpu_leaky_relu, mps_leaky_relu.to('cpu'))

        # test backward pass
        cpu_grad = torch.ones_like(cpu_leaky_relu)
        mps_grad = cpu_grad.to('mps')
        cpu_leaky_relu.backward(gradient=cpu_grad)
        mps_leaky_relu.backward(gradient=mps_grad)
        torch.testing.assert_allclose(cpu_x.grad, mps_x.grad.to('cpu'))
//  [python]
def _testLeakyRelu(self, np_features, negative_slope, device):
        cpu_x = torch.from_numpy(np_features).requires_grad_()
        mps_x = torch.from_numpy(np_features).to('mps').requires_grad_()
        relu_op = torch.nn.LeakyReLU(negative_slope)

        cpu_leaky_relu = relu_op(cpu_x)
        mps_leaky_relu = relu_op(mps_x)
        torch.testing.assert_close(cpu_leaky_relu, mps_leaky_relu.to('cpu'))

        # test backward pass
        cpu_grad = torch.ones_like(cpu_leaky_relu)
        mps_grad = cpu_grad.to('mps')
        cpu_leaky_relu.backward(gradient=cpu_grad)
        mps_leaky_relu.backward(gradient=mps_grad)
        torch.testing.assert_close(cpu_x.grad, mps_x.grad.to('cpu'))
//  [python]
def test_addmm(self):
        A = torch.ones(5, 5).to("mps")
        B = torch.ones(5, 6).to("mps")
        C = torch.ones(6, 5).to("mps")
        D = torch.addmm(A, B, C).to("cpu")
        torch.testing.assert_allclose(D, torch.full((5, 5), 7.0))
//  [python]
def test_addmm(self):
        A = torch.ones(5, 5).to("mps")
        B = torch.ones(5, 6).to("mps")
        C = torch.ones(6, 5).to("mps")
        D = torch.addmm(A, B, C).to("cpu")
        torch.testing.assert_close(D, torch.full((5, 5), 7.0))
//  [python]
def test_addr(self):
        A = torch.ones(5, 10).to("mps")
        B = torch.ones(5).to("mps")
        C = torch.ones(10).to("mps")
        D = torch.addr(A, B, C).to("cpu")
        torch.testing.assert_allclose(D, torch.full((5, 10), 2.0))
//  [python]
def test_addr(self):
        A = torch.ones(5, 10).to("mps")
        B = torch.ones(5).to("mps")
        C = torch.ones(10).to("mps")
        D = torch.addr(A, B, C).to("cpu")
        torch.testing.assert_close(D, torch.full((5, 10), 2.0))
//  [python]
def test_local_scalar_dense_mps(self):
        x_cpu = torch.randn(1)
        y_mps = x_cpu.to("mps")
        self.assertEqual(x_cpu.item(), y_mps.item())
//  [python]
def test_local_scalar_dense_mps(self):
        x_cpu = torch.randn(1)
        y_mps = x_cpu.to("mps")
        torch.testing.assert_close(x_cpu.item(), y_mps.item())
//  [python]
def test_slice_scatter(self):
        shape = (4, 4)
        tensor = torch.randint(10, shape, device="mps")
        tensor_before = tensor.clone()
        torch.empty(shape[0], shape[1] * 2, device="mps")[:, ::2].copy_(tensor)
        self.assertEqual(tensor, tensor_before)
//  [python]
def test_slice_scatter(self):
        shape = (4, 4)
        tensor = torch.randint(10, shape, device="mps")
        tensor_before = tensor.clone()
        torch.empty(shape[0], shape[1] * 2, device="mps")[:, ::2].copy_(tensor)
        torch.testing.assert_close(tensor, tensor_before)
//  [python]
def test_assert_close(self):
        a = torch.ones(1, device="mps")
        b = torch.zeros(1, device="mps")
        inf = a / b
        nan = b / b

        with self.assertRaisesRegex(AssertionError, "Tensor-likes are not close!"):
            torch.testing.assert_allclose(a, inf)
//  [python]
def test_assert_close(self):
        a = torch.ones(1, device="mps")
        b = torch.zeros(1, device="mps")
        inf = a / b
        nan = b / b

        with self.assertRaisesRegex(AssertionError, "Tensor-likes are not close!"):
            torch.testing.assert_close(a, inf)
//  [python]
def assert_close(a, b):
    rtol, atol = {
        torch.float16: (5e-3, 5e-3),
        torch.bfloat16: (3e-2, 2e-2),
    }[a.dtype]
    t.assert_close(a, b, rtol=rtol, atol=atol)
//  [python]
def assert_close(a, b):
    rtol, atol = {
        torch.float16: (5e-3, 5e-3),
        torch.bfloat16: (3e-2, 2e-2),
    }[a.dtype]
    torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
//  [python]
def test_preprocess_obs_discrete():
    actual = preprocess_obs(torch.tensor([2], dtype=torch.long), spaces.Discrete(3))
    expected = torch.tensor([[0.0, 0.0, 1.0]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_discrete():
    actual = preprocess_obs(torch.tensor([2], dtype=torch.long), spaces.Discrete(3))
    expected = torch.tensor([[0.0, 0.0, 1.0]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_preprocess_obs_multibinary():
    actual = preprocess_obs(torch.tensor([[1, 0, 1]], dtype=torch.long), spaces.MultiBinary(3))
    expected = torch.tensor([[1.0, 0.0, 1.0]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_multibinary():
    actual = preprocess_obs(torch.tensor([[1, 0, 1]], dtype=torch.long), spaces.MultiBinary(3))
    expected = torch.tensor([[1.0, 0.0, 1.0]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_preprocess_obs_multidiscrete():
    actual = preprocess_obs(torch.tensor([[2, 0]], dtype=torch.long), spaces.MultiDiscrete([3, 2]))
    expected = torch.tensor([[0.0, 0.0, 1.0, 1.0, 0.0]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_multidiscrete():
    actual = preprocess_obs(torch.tensor([[2, 0]], dtype=torch.long), spaces.MultiDiscrete([3, 2]))
    expected = torch.tensor([[0.0, 0.0, 1.0, 1.0, 0.0]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_preprocess_obs_box():
    actual = preprocess_obs(torch.tensor([[1.5, 0.3, -1.8]], dtype=torch.float32), spaces.Box(-2, 2, shape=(3,)))
    expected = torch.tensor([[1.5, 0.3, -1.8]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_box():
    actual = preprocess_obs(torch.tensor([[1.5, 0.3, -1.8]], dtype=torch.float32), spaces.Box(-2, 2, shape=(3,)))
    expected = torch.tensor([[1.5, 0.3, -1.8]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_preprocess_obs_multidimensional_multibinary():
    actual = preprocess_obs(torch.tensor([[[1, 0], [1, 1], [0, 1]]], dtype=torch.long), spaces.MultiBinary([3, 2]))
    expected = torch.tensor([[[1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_multidimensional_multibinary():
    actual = preprocess_obs(torch.tensor([[[1, 0], [1, 1], [0, 1]]], dtype=torch.long), spaces.MultiBinary([3, 2]))
    expected = torch.tensor([[[1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_preprocess_obs_multidimensional_box():
    actual = preprocess_obs(
        torch.tensor([[[1.5, 0.3, -1.8], [0.1, -0.6, -1.4]]], dtype=torch.float32), spaces.Box(-2, 2, shape=(3, 2))
    )
    expected = torch.tensor([[[1.5, 0.3, -1.8], [0.1, -0.6, -1.4]]], dtype=torch.float32)
    torch.testing.assert_allclose(actual, expected)
//  [python]
def test_preprocess_obs_multidimensional_box():
    actual = preprocess_obs(
        torch.tensor([[[1.5, 0.3, -1.8], [0.1, -0.6, -1.4]]], dtype=torch.float32), spaces.Box(-2, 2, shape=(3, 2))
    )
    expected = torch.tensor([[[1.5, 0.3, -1.8], [0.1, -0.6, -1.4]]], dtype=torch.float32)
    torch.testing.assert_close(actual, expected)
//  [python]
def test_recreate_nets_parameters_shrink_preserved(device):
    network = nn.Sequential(
        nn.Linear(4, 32), nn.ReLU(), nn.Linear(32, 32), nn.ReLU(), nn.Linear(32, 2)
    )

    input_tensor = torch.randn(1, 4)
    evolvable_network = MakeEvolvable(network, input_tensor, device=device)

    feature_net = evolvable_network.feature_net
    feature_net_dict = dict(feature_net.named_parameters())

    # Modify the architecture
    evolvable_network.hidden_size = evolvable_network.hidden_size[:-1]
    evolvable_network.recreate_nets(shrink_params=True)
    new_feature_net = evolvable_network.feature_net

    for key, param in new_feature_net.named_parameters():
        if key in feature_net_dict.keys():
            torch.allclose(param, feature_net_dict[key])
//  [python]
def test_recreate_nets_parameters_shrink_preserved(device):
    network = nn.Sequential(
        nn.Linear(4, 32), nn.ReLU(), nn.Linear(32, 32), nn.ReLU(), nn.Linear(32, 2)
    )

    input_tensor = torch.randn(1, 4)
    evolvable_network = MakeEvolvable(network, input_tensor, device=device)

    feature_net = evolvable_network.feature_net
    feature_net_dict = dict(feature_net.named_parameters())

    # Modify the architecture
    evolvable_network.hidden_size = evolvable_network.hidden_size[:-1]
    evolvable_network.recreate_nets(shrink_params=True)
    new_feature_net = evolvable_network.feature_net

    for key, param in new_feature_net.named_parameters():
        if key in feature_net_dict.keys():
            torch.testing.assert_close(param, feature_net_dict[key])
//  [python]
def test_sem_distribution_passthrough():
    """Test distribution properties."""
    shapes = {"a": torch.Size([5]), "b": torch.Size([2])}
    adjacency_dist, noise_dist, func = create_sem_params(shapes)
    sem_dist = SEMDistribution(adjacency_dist, noise_dist, func)
    t.assert_close(adjacency_dist.entropy(), sem_dist.entropy())
    t.assert_close(adjacency_dist.mean, sem_dist.mean.graph)
    t.assert_close(adjacency_dist.mode, sem_dist.mode.graph)
    t.assert_close(adjacency_dist.log_prob(adjacency_dist.mode), sem_dist.log_prob(sem_dist.mode))
//  [python]
def test_sem_distribution_passthrough():
    """Test distribution properties."""
    shapes = {"a": torch.Size([5]), "b": torch.Size([2])}
    adjacency_dist, noise_dist, func = create_sem_params(shapes)
    sem_dist = SEMDistribution(adjacency_dist, noise_dist, func)
    torch.testing.assert_close(adjacency_dist.entropy(), sem_dist.entropy())
    torch.testing.assert_close(adjacency_dist.mean, sem_dist.mean.graph)
    torch.testing.assert_close(adjacency_dist.mode, sem_dist.mode.graph)
    torch.testing.assert_close(adjacency_dist.log_prob(adjacency_dist.mode), sem_dist.log_prob(sem_dist.mode))
//  [python]
def test_remove_mlp_layer_simple(simple_mlp_2, device):
    input_tensor = torch.randn(1, 10)
    evolvable_network = MakeEvolvable(simple_mlp_2, input_tensor, device=device)
    feature_net = evolvable_network.feature_net
    feature_net_dict = dict(feature_net.named_parameters())
    initial_num_layers = len(evolvable_network.hidden_size)
    assert evolvable_network.mlp_layer_info == {
        "activation_layers": {0: "ReLU", 1: "ReLU"}
    }
    evolvable_network.remove_mlp_layer()
    new_feature_net = evolvable_network.feature_net
    assert len(evolvable_network.hidden_size) == initial_num_layers - 1
    assert evolvable_network.mlp_layer_info == {
        "activation_layers": {0: "ReLU"}
    }, evolvable_network.mlp_layer_info
    for key, param in new_feature_net.named_parameters():
        if key in feature_net_dict.keys():
            torch.testing.assert_allclose(param, feature_net_dict[key])
//  [python]
def test_remove_mlp_layer_simple(simple_mlp_2, device):
    input_tensor = torch.randn(1, 10)
    evolvable_network = MakeEvolvable(simple_mlp_2, input_tensor, device=device)
    feature_net = evolvable_network.feature_net
    feature_net_dict = dict(feature_net.named_parameters())
    initial_num_layers = len(evolvable_network.hidden_size)
    assert evolvable_network.mlp_layer_info == {
        "activation_layers": {0: "ReLU", 1: "ReLU"}
    }
    evolvable_network.remove_mlp_layer()
    new_feature_net = evolvable_network.feature_net
    assert len(evolvable_network.hidden_size) == initial_num_layers - 1
    assert evolvable_network.mlp_layer_info == {
        "activation_layers": {0: "ReLU"}
    }, evolvable_network.mlp_layer_info
    for key, param in new_feature_net.named_parameters():
        if key in feature_net_dict.keys():
            torch.testing.assert_close(param, feature_net_dict[key])
//  [python]
def test_draw_boxes_dtypes(fill):
    img_uint8 = torch.full((3, 100, 100), 255, dtype=torch.uint8)
    out_uint8 = utils.draw_bounding_boxes(img_uint8, boxes, fill=fill)

    assert img_uint8 is not out_uint8
    assert out_uint8.dtype == torch.uint8

    img_float = to_dtype(img_uint8, torch.float, scale=True)
    out_float = utils.draw_bounding_boxes(img_float, boxes, fill=fill)

    assert img_float is not out_float
    assert out_float.is_floating_point()

    torch.testing.assert_allclose(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def test_draw_boxes_dtypes(fill):
    img_uint8 = torch.full((3, 100, 100), 255, dtype=torch.uint8)
    out_uint8 = utils.draw_bounding_boxes(img_uint8, boxes, fill=fill)

    assert img_uint8 is not out_uint8
    assert out_uint8.dtype == torch.uint8

    img_float = to_dtype(img_uint8, torch.float, scale=True)
    out_float = utils.draw_bounding_boxes(img_float, boxes, fill=fill)

    assert img_float is not out_float
    assert out_float.is_floating_point()

    torch.testing.assert_close(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def test_gather(self, input, dim, index):
        model = _TestModel(dim)
        dummy_input = torch.zeros_like(input)
        dummy_index = torch.zeros_like(index)
        trt_model = module2trt(model, args=[dummy_input, dummy_index])

        with torch.inference_mode():
            gt = model(input, index)
            out = trt_model(input, index)
        torch.testing.assert_allclose(out, gt)
//  [python]
def test_gather(self, input, dim, index):
        model = _TestModel(dim)
        dummy_input = torch.zeros_like(input)
        dummy_index = torch.zeros_like(index)
        trt_model = module2trt(model, args=[dummy_input, dummy_index])

        with torch.inference_mode():
            gt = model(input, index)
            out = trt_model(input, index)
        torch.testing.assert_close(out, gt)
//  [python]
def test_draw_keypoints_dtypes():
    image_uint8 = torch.randint(0, 256, size=(3, 100, 100), dtype=torch.uint8)
    image_float = to_dtype(image_uint8, torch.float, scale=True)

    out_uint8 = utils.draw_keypoints(image_uint8, keypoints)
    out_float = utils.draw_keypoints(image_float, keypoints)

    assert out_uint8.dtype == torch.uint8
    assert out_uint8 is not image_uint8

    assert out_float.is_floating_point()
    assert out_float is not image_float

    torch.testing.assert_allclose(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def test_draw_keypoints_dtypes():
    image_uint8 = torch.randint(0, 256, size=(3, 100, 100), dtype=torch.uint8)
    image_float = to_dtype(image_uint8, torch.float, scale=True)

    out_uint8 = utils.draw_keypoints(image_uint8, keypoints)
    out_float = utils.draw_keypoints(image_float, keypoints)

    assert out_uint8.dtype == torch.uint8
    assert out_uint8 is not image_uint8

    assert out_float.is_floating_point()
    assert out_float is not image_float

    torch.testing.assert_close(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def test_reciprocal_rank_with_valid_input(self) -> None:
        input = torch.tensor(
            [
                [0.4826, 0.9517, 0.8967, 0.8995, 0.1584, 0.9445, 0.9700],
                [0.4938, 0.7517, 0.8039, 0.7167, 0.9488, 0.9607, 0.7091],
                [0.5127, 0.4732, 0.5461, 0.5617, 0.9198, 0.0847, 0.2337],
                [0.4175, 0.9452, 0.9852, 0.2131, 0.5016, 0.7305, 0.0516],
            ]
        )
        target = torch.tensor([3, 5, 2, 1])

        t.assert_allclose(
            reciprocal_rank(input, target, k=None),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        t.assert_allclose(
            reciprocal_rank(input, target, k=1),
            torch.tensor([0.0000, 1.0000, 0.0000, 0.0000]),
        )
        t.assert_allclose(
            reciprocal_rank(input, target, k=3),
            torch.tensor([0.0000, 1.0000, 1.0000 / 3, 0.5000]),
        )
        t.assert_allclose(
            reciprocal_rank(input, target, k=5),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        t.assert_allclose(
            reciprocal_rank(input, target, k=20),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        t.assert_allclose(
            reciprocal_rank(input, target, k=100),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
//  [python]
def test_reciprocal_rank_with_valid_input(self) -> None:
        input = torch.tensor(
            [
                [0.4826, 0.9517, 0.8967, 0.8995, 0.1584, 0.9445, 0.9700],
                [0.4938, 0.7517, 0.8039, 0.7167, 0.9488, 0.9607, 0.7091],
                [0.5127, 0.4732, 0.5461, 0.5617, 0.9198, 0.0847, 0.2337],
                [0.4175, 0.9452, 0.9852, 0.2131, 0.5016, 0.7305, 0.0516],
            ]
        )
        target = torch.tensor([3, 5, 2, 1])

        torch.testing.assert_close(
            reciprocal_rank(input, target, k=None),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        torch.testing.assert_close(
            reciprocal_rank(input, target, k=1),
            torch.tensor([0.0000, 1.0000, 0.0000, 0.0000]),
        )
        torch.testing.assert_close(
            reciprocal_rank(input, target, k=3),
            torch.tensor([0.0000, 1.0000, 1.0000 / 3, 0.5000]),
        )
        torch.testing.assert_close(
            reciprocal_rank(input, target, k=5),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        torch.testing.assert_close(
            reciprocal_rank(input, target, k=20),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
        torch.testing.assert_close(
            reciprocal_rank(input, target, k=100),
            torch.tensor([0.2500, 1.0000, 1.0000 / 3, 0.5000]),
        )
//  [python]
def test_draw_segmentation_masks_dtypes():
    num_masks, h, w = 2, 100, 100

    masks = torch.randint(0, 2, (num_masks, h, w), dtype=torch.bool)

    img_uint8 = torch.randint(0, 256, size=(3, h, w), dtype=torch.uint8)
    out_uint8 = utils.draw_segmentation_masks(img_uint8, masks)

    assert img_uint8 is not out_uint8
    assert out_uint8.dtype == torch.uint8

    img_float = to_dtype(img_uint8, torch.float, scale=True)
    out_float = utils.draw_segmentation_masks(img_float, masks)

    assert img_float is not out_float
    assert out_float.is_floating_point()

    torch.testing.assert_allclose(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def test_draw_segmentation_masks_dtypes():
    num_masks, h, w = 2, 100, 100

    masks = torch.randint(0, 2, (num_masks, h, w), dtype=torch.bool)

    img_uint8 = torch.randint(0, 256, size=(3, h, w), dtype=torch.uint8)
    out_uint8 = utils.draw_segmentation_masks(img_uint8, masks)

    assert img_uint8 is not out_uint8
    assert out_uint8.dtype == torch.uint8

    img_float = to_dtype(img_uint8, torch.float, scale=True)
    out_float = utils.draw_segmentation_masks(img_float, masks)

    assert img_float is not out_float
    assert out_float.is_floating_point()

    torch.testing.assert_close(out_uint8, to_dtype(out_float, torch.uint8, scale=True), rtol=0, atol=1)
//  [python]
def verify_grads(ref_model, path_and_grads, tol=1e-5):
    for path, grad in path_and_grads.items():
        param = ref_model
        for token in path.split("."):
            param = getattr(param, token)
        torch.allclose(
            grad,
            param.grad,
            # pylint: disable=cell-var-from-loop
            atol=tol,
            rtol=tol,
        )
        print(f"{path}.grad verified")
//  [python]
def verify_grads(ref_model, path_and_grads, tol=1e-5):
    for path, grad in path_and_grads.items():
        param = ref_model
        for token in path.split("."):
            param = getattr(param, token)
        torch.testing.assert_close(
            grad,
            param.grad,
            # pylint: disable=cell-var-from-loop
            msg=lambda msg: f"{path}.grad mismatch\n{msg}",
            atol=tol,
            rtol=tol,
        )
        print(f"{path}.grad verified")
//  [python]
def test_glu():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.glu import GLU

    batch_size, seq_len, feature_dim, head_num = 7, 12, 16, 4
    x = torch.rand((batch_size, seq_len, feature_dim))
    model_pytorch = nn.GLU(dim=-1)
    model_pytorch.eval()
    out_pytorch = model_pytorch(x)

    model_brocolli = GLU(dim=-1)
    model_brocolli.eval()

    out_brocolli = model_brocolli(x)

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_glu():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.glu import GLU

    batch_size, seq_len, feature_dim, head_num = 7, 12, 16, 4
    x = torch.rand((batch_size, seq_len, feature_dim))
    model_pytorch = nn.GLU(dim=-1)
    model_pytorch.eval()
    out_pytorch = model_pytorch(x)

    model_brocolli = GLU(dim=-1)
    model_brocolli.eval()

    out_brocolli = model_brocolli(x)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_encoder_layer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerEncoderLayer
    from brocolli.converter.pytorch_layer.utils import transform_weight

    model_pytorch = nn.TransformerEncoderLayer(d_model=512, nhead=8, batch_first=False)
    model_pytorch.eval()

    src = torch.rand(32, 10, 512)
    out_pytorch = model_pytorch(src)

    model_brocolli = TransformerEncoderLayer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(src)

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_encoder_layer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerEncoderLayer
    from brocolli.converter.pytorch_layer.utils import transform_weight

    model_pytorch = nn.TransformerEncoderLayer(d_model=512, nhead=8, batch_first=False)
    model_pytorch.eval()

    src = torch.rand(32, 10, 512)
    out_pytorch = model_pytorch(src)

    model_brocolli = TransformerEncoderLayer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(src)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_layernorm(normalized_shape, elementwise_affine):
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.layernorm import LayerNorm

    N, C, H, W = 20, 5, 10, 10
    embedding = torch.randn(N, C, H, W)
    model_pytorch = nn.LayerNorm(
        normalized_shape, elementwise_affine=elementwise_affine
    )
    model_pytorch.eval()

    out_pytorch = model_pytorch(embedding)

    model_brocolli = LayerNorm.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(embedding)

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_layernorm(normalized_shape, elementwise_affine):
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.layernorm import LayerNorm

    N, C, H, W = 20, 5, 10, 10
    embedding = torch.randn(N, C, H, W)
    model_pytorch = nn.LayerNorm(
        normalized_shape, elementwise_affine=elementwise_affine
    )
    model_pytorch.eval()

    out_pytorch = model_pytorch(embedding)

    model_brocolli = LayerNorm.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(embedding)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_decoder_layer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerDecoderLayer

    model_pytorch = nn.TransformerDecoderLayer(d_model=512, nhead=8, batch_first=False)
    model_pytorch.eval()

    memory = torch.rand(10, 32, 512)
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(tgt, memory)

    model_brocolli = TransformerDecoderLayer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(tgt, memory)

    tol = 1e-5
    torch.allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_decoder_layer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerDecoderLayer

    model_pytorch = nn.TransformerDecoderLayer(d_model=512, nhead=8, batch_first=False)
    model_pytorch.eval()

    memory = torch.rand(10, 32, 512)
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(tgt, memory)

    model_brocolli = TransformerDecoderLayer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(tgt, memory)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_encoder():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerEncoder

    encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8)
    model_pytorch = nn.TransformerEncoder(encoder_layer, num_layers=2)
    model_pytorch.eval()

    src = torch.rand(10, 32, 512)
    out_pytorch = model_pytorch(src)

    model_brocolli = TransformerEncoder.from_torch(model_pytorch)

    model_brocolli.eval()

    out_brocolli = model_brocolli(src)

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_encoder():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerEncoder

    encoder_layer = nn.TransformerEncoderLayer(d_model=512, nhead=8)
    model_pytorch = nn.TransformerEncoder(encoder_layer, num_layers=2)
    model_pytorch.eval()

    src = torch.rand(10, 32, 512)
    out_pytorch = model_pytorch(src)

    model_brocolli = TransformerEncoder.from_torch(model_pytorch)

    model_brocolli.eval()

    out_brocolli = model_brocolli(src)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_mha():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.mha import MultiheadAttention
    from brocolli.converter.pytorch_layer.utils import transform_weight

    batch_size, seq_len, feature_dim, head_num = 7, 12, 16, 4
    q = torch.rand((batch_size, seq_len, feature_dim))
    k = torch.rand((batch_size, seq_len, feature_dim))
    v = torch.rand((batch_size, seq_len, feature_dim))

    model_pytorch = nn.MultiheadAttention(feature_dim, head_num)
    model_pytorch.eval()
    out_pytorch = model_pytorch(q, k, v)[0]

    state_dict = transform_weight(model_pytorch)
    model_brocolli = MultiheadAttention(feature_dim, head_num)
    model_brocolli.load_state_dict(state_dict)
    model_brocolli.eval()

    out_brocolli = model_brocolli(q, k, v)[0]

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_mha():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.mha import MultiheadAttention
    from brocolli.converter.pytorch_layer.utils import transform_weight

    batch_size, seq_len, feature_dim, head_num = 7, 12, 16, 4
    q = torch.rand((batch_size, seq_len, feature_dim))
    k = torch.rand((batch_size, seq_len, feature_dim))
    v = torch.rand((batch_size, seq_len, feature_dim))

    model_pytorch = nn.MultiheadAttention(feature_dim, head_num)
    model_pytorch.eval()
    out_pytorch = model_pytorch(q, k, v)[0]

    state_dict = transform_weight(model_pytorch)
    model_brocolli = MultiheadAttention(feature_dim, head_num)
    model_brocolli.load_state_dict(state_dict)
    model_brocolli.eval()

    out_brocolli = model_brocolli(q, k, v)[0]

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_decoder():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerDecoder

    decoder_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8)
    model_pytorch = nn.TransformerDecoder(decoder_layer, num_layers=2)
    model_pytorch.eval()

    memory = torch.rand(10, 32, 512)
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(tgt, memory)

    model_brocolli = TransformerDecoder.from_torch(model_pytorch)

    model_brocolli.eval()

    out_brocolli = model_brocolli(tgt, memory)

    tol = 1e-5
    torch.testing.assert_allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer_decoder():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import TransformerDecoder

    decoder_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8)
    model_pytorch = nn.TransformerDecoder(decoder_layer, num_layers=2)
    model_pytorch.eval()

    memory = torch.rand(10, 32, 512)
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(tgt, memory)

    model_brocolli = TransformerDecoder.from_torch(model_pytorch)

    model_brocolli.eval()

    out_brocolli = model_brocolli(tgt, memory)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def torch_assert_equal(actual, expected, **kwargs):
    """
    compare two tensors or non-tensor numbers for their equality
    """
    # assert_close was added around pt-1.9, it does better checks - e.g will check dimensions match
    return torch.testing.assert_allclose(actual, expected, rtol=0.0, atol=0.0, **kwargs)
//  [python]
def torch_assert_equal(actual, expected, **kwargs):
    """
    compare two tensors or non-tensor numbers for their equality
    """
    # assert_close was added around pt-1.9, it does better checks - e.g will check dimensions match
    return torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0, **kwargs)
//  [python]
def test_transformer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import Transformer

    model_pytorch = nn.Transformer(nhead=16, num_encoder_layers=2, num_decoder_layers=2)
    model_pytorch.eval()

    src = torch.rand((10, 32, 512))
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(src, tgt)

    model_brocolli = Transformer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(src, tgt)

    tol = 1e-5
    torch.allclose(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def test_transformer():
    import torch
    import torch.nn as nn
    from brocolli.converter.pytorch_layer.transformer import Transformer

    model_pytorch = nn.Transformer(nhead=16, num_encoder_layers=2, num_decoder_layers=2)
    model_pytorch.eval()

    src = torch.rand((10, 32, 512))
    tgt = torch.rand(20, 32, 512)
    out_pytorch = model_pytorch(src, tgt)

    model_brocolli = Transformer.from_torch(model_pytorch)
    model_brocolli.eval()

    out_brocolli = model_brocolli(src, tgt)

    tol = 1e-5
    torch.testing.assert_close(out_pytorch, out_brocolli, rtol=tol, atol=tol)
//  [python]
def torch_assert_close(actual, expected, **kwargs):
    """
    compare two tensors or non-tensor numbers for their closeness.
    """
    # assert_close was added around pt-1.9, it does better checks - e.g will check dimensions match
    return torch.testing.assert_close(actual, expected, **kwargs)
//  [python]
def test_input_tensor_addition(jit):
    def foo(a, b):
        return a + 2 + b

    jfoo = jit(foo)

    args = (4, 3)

    thunder_result = jfoo(*args)
    python_result = foo(*args)

    t.assert_close(thunder_result, python_result)
//  [python]
def test_input_tensor_addition(jit):
    def foo(a, b):
        return a + 2 + b

    jfoo = jit(foo)

    args = (4, 3)

    thunder_result = jfoo(*args)
    python_result = foo(*args)

    assert_close(thunder_result, python_result)
//  [python]
def test_constant_if(jit):
    def foo(a, b):
        if 3 < 5:
            return a + b
        else:
            assert False

    jfoo = jit(foo)

    args = (4, 3)

    thunder_result = jfoo(*args)
    python_result = foo(*args)

    t.assert_close(thunder_result, python_result)
//  [python]
def test_constant_if(jit):
    def foo(a, b):
        if 3 < 5:
            return a + b
        else:
            assert False

    jfoo = jit(foo)

    args = (4, 3)

    thunder_result = jfoo(*args)
    python_result = foo(*args)

    assert_close(thunder_result, python_result)
//  [python]
def test_nested_function_call(jit):
    jitting = False

    def bar(a, b):
        assert is_jitting_with_raise() == jitting
        return a + b

    def foo(a, b):
        assert is_jitting_with_raise() == jitting
        return bar(a + 1, b)

    jfoo = jit(foo)
    args = (4, 3)

    python_result = foo(*args)
    jitting = True
    thunder_result = jfoo(*args)

    t.assert_close(thunder_result, python_result)
//  [python]
def test_nested_function_call(jit):
    jitting = False

    def bar(a, b):
        assert is_jitting_with_raise() == jitting
        return a + b

    def foo(a, b):
        assert is_jitting_with_raise() == jitting
        return bar(a + 1, b)

    jfoo = jit(foo)
    args = (4, 3)

    python_result = foo(*args)
    jitting = True
    thunder_result = jfoo(*args)

    assert_close(thunder_result, python_result)
//  [python]
def test_trace_buffer_slice(self):
        bs, d_hid = 10, 23

        class ExampleCode(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.mm_param = torch.nn.Parameter(torch.randn(d_hid, d_hid))
                self.mm_param2 = torch.nn.Parameter(torch.randn(d_hid, d_hid))
                self.lin = torch.nn.Linear(d_hid, d_hid)
                self.register_buffer('buffer', torch.randn(bs + 100, d_hid))

            def forward(self, x):
                x = torch.mm(x, self.mm_param)
                skip_connection = x
                x = torch.relu(x)
                x = torch.mm(x, self.mm_param) + self.buffer[:x.shape[0]]
                x = self.lin(x)
                x = torch.relu(x)
                x = x + skip_connection
                x = torch.mm(x, self.mm_param2)
                x = self.lin(x)
                return x

        ec = ExampleCode()

        traced = torch.fx.symbolic_trace(ec)

        x = torch.randn(bs, d_hid)
        torch.testing.assert_allclose(ec(x), traced(x))