jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def test_optimizer_lr_one_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1 = 0.1

    named_params_or_groups = []
    for name, param in model.named_parameters():
        named_params_or_groups.append((name, param))
    named_params_or_groups = [{"named_params": named_params_or_groups, "lr": lr1}]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        fc1_grad = model.fc1.weight.grad.clone()
        fc2_grad = model.fc2.weight.grad.clone()

        # compute gradient manually
        with torch.no_grad():
            expected_fc1_weight = model.fc1.weight - lr1 * fc1_grad
            expected_fc2_weight = model.fc2.weight - lr1 * fc2_grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        torch.testing.assert_close(expected_fc1_weight, updated_fc1_weight)
        torch.testing.assert_close(expected_fc2_weight, updated_fc2_weight)
//  [python]
def test_log_survival_function(self):
        t.assert_close(
            UniformNoise(self.distribution).log_survival_function(self.x),
            ncF.log_survival_function(self.x, self.distribution),
        )
//  [python]
def test_log_survival_function(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).log_survival_function(self.x),
            ncF.log_survival_function(self.x, self.distribution),
        )
//  [python]
def test_optimizer_lr_weight_decay_one_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1 = 0.1
    weight_decay = 0.1

    named_params_or_groups = []
    for name, param in model.named_parameters():
        named_params_or_groups.append((name, param))
    named_params_or_groups = [{"named_params": named_params_or_groups, "lr": lr1, "weight_decay": weight_decay}]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        # Compute gradient manually and apply weight decay
        with torch.no_grad():
            expected_fc1_weight = (1 - lr1 * weight_decay) * model.fc1.weight - lr1 * model.fc1.weight.grad
            expected_fc2_weight = (1 - lr1 * weight_decay) * model.fc2.weight - lr1 * model.fc2.weight.grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        t.assert_close(expected_fc1_weight, updated_fc1_weight)
        t.assert_close(expected_fc2_weight, updated_fc2_weight)
//  [python]
def test_optimizer_lr_weight_decay_one_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1 = 0.1
    weight_decay = 0.1

    named_params_or_groups = []
    for name, param in model.named_parameters():
        named_params_or_groups.append((name, param))
    named_params_or_groups = [{"named_params": named_params_or_groups, "lr": lr1, "weight_decay": weight_decay}]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        # Compute gradient manually and apply weight decay
        with torch.no_grad():
            expected_fc1_weight = (1 - lr1 * weight_decay) * model.fc1.weight - lr1 * model.fc1.weight.grad
            expected_fc2_weight = (1 - lr1 * weight_decay) * model.fc2.weight - lr1 * model.fc2.weight.grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        torch.testing.assert_close(expected_fc1_weight, updated_fc1_weight)
        torch.testing.assert_close(expected_fc2_weight, updated_fc2_weight)
//  [python]
def test_optimizer_lr_multiple_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1, lr2 = 0.1, 0.001

    named_params_or_groups = [
        {"named_params": [(name, param) for name, param in model.named_parameters() if "fc1" in name], "lr": lr1},
        {"named_params": [(name, param) for name, param in model.named_parameters() if "fc2" in name], "lr": lr2},
    ]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        fc1_grad = model.fc1.weight.grad.clone()
        fc2_grad = model.fc2.weight.grad.clone()

        with torch.no_grad():
            expected_fc1_weight = model.fc1.weight - lr1 * fc1_grad
            expected_fc2_weight = model.fc2.weight - lr2 * fc2_grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        t.assert_close(expected_fc1_weight, updated_fc1_weight)
        t.assert_close(expected_fc2_weight, updated_fc2_weight)
//  [python]
def test_optimizer_lr_multiple_group():
    set_random_seed(42)

    model = DummyModel().to("cuda")

    lr1, lr2 = 0.1, 0.001

    named_params_or_groups = [
        {"named_params": [(name, param) for name, param in model.named_parameters() if "fc1" in name], "lr": lr1},
        {"named_params": [(name, param) for name, param in model.named_parameters() if "fc2" in name], "lr": lr2},
    ]

    optimizer = NamedOptimizer(
        named_params_or_groups=named_params_or_groups,
        optimizer_builder=lambda param_groups: optim.SGD(
            param_groups,
            lr=9999999,  # this is a dummy value that should be overwritten by the lr in the named_params_or_groups
        ),
    )

    input = torch.randn(10, 10).to(device="cuda")
    target = torch.randint(0, 2, (10,)).to(device="cuda")

    for _ in range(100):
        optimizer.zero_grad()

        output = model(input)
        loss = F.cross_entropy(output, target)
        loss.backward()

        fc1_grad = model.fc1.weight.grad.clone()
        fc2_grad = model.fc2.weight.grad.clone()

        with torch.no_grad():
            expected_fc1_weight = model.fc1.weight - lr1 * fc1_grad
            expected_fc2_weight = model.fc2.weight - lr2 * fc2_grad

        optimizer.step()

        updated_fc1_weight = model.fc1.weight
        updated_fc2_weight = model.fc2.weight

        torch.testing.assert_close(expected_fc1_weight, updated_fc1_weight)
        torch.testing.assert_close(expected_fc2_weight, updated_fc2_weight)
//  [python]
def test_mean(self):
        self.assertEqual(
            UniformNoise(self.distribution).mean,
            torch.tensor(0.0),
        )
//  [python]
def test_mean(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).mean,
            torch.tensor(0.0),
        )
//  [python]
def test_lower_tail(self):
        self.assertEqual(
            UniformNoise(self.distribution).lower_tail(1.0),
            ncF.lower_tail(self.distribution, 1.0),
        )
//  [python]
def test_lower_tail(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).lower_tail(1.0),
            ncF.lower_tail(self.distribution, 1.0),
        )
//  [python]
def test_prob(self):
        self.assertEqual(
            UniformNoise(self.distribution).prob(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_prob(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).prob(self.x).shape,
            torch.Size([32]),
        )
//  [python]
def test_quantization_offset(self):
        self.assertEqual(
            UniformNoise(self.distribution).quantization_offset,
            torch.tensor(0.0),
        )
//  [python]
def test_quantization_offset(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).quantization_offset,
            torch.tensor(0.0),
        )
//  [python]
def test_rsample(self):
        self.assertEqual(
            UniformNoise(self.distribution).rsample(torch.Size((5, 4))).shape,
            torch.Size([5, 4]),
        )
//  [python]
def test_rsample(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).rsample(torch.Size((5, 4))).shape,
            torch.Size([5, 4]),
        )
//  [python]
def test_survival_function(self):
        t.assert_close(
            UniformNoise(self.distribution).survival_function(self.x),
            ncF.survival_function(self.x, self.distribution),
        )
//  [python]
def test_survival_function(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).survival_function(self.x),
            ncF.survival_function(self.x, self.distribution),
        )
//  [python]
def test_variance(self):
        self.assertEqual(
            UniformNoise(self.distribution).variance,
            torch.tensor(1.0),
        )
//  [python]
def test_variance(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).variance,
            torch.tensor(1.0),
        )
//  [python]
def test_upper_tail(self):
        self.assertEqual(
            UniformNoise(self.distribution).upper_tail(1.0),
            ncF.upper_tail(self.distribution, 1.0),
        )
//  [python]
def test_upper_tail(self):
        torch.testing.assert_close(
            UniformNoise(self.distribution).upper_tail(1.0),
            ncF.upper_tail(self.distribution, 1.0),
        )
//  [python]
def test_log_expm1():
    rng = numpy.random.default_rng(0xFEEEFEEE)

    x = rng.random((32,), dtype=numpy.float64)

    actual = log_expm1(torch.tensor(x, dtype=torch.float))

    assert torch.isfinite(actual).all()

    torch.testing.assert_allclose(
        actual,
        torch.tensor(numpy.log(numpy.expm1(x)), dtype=torch.float),
    )
//  [python]
def test_log_expm1():
    rng = numpy.random.default_rng(0xFEEEFEEE)

    x = rng.random((32,), dtype=numpy.float64)

    actual = log_expm1(torch.tensor(x, dtype=torch.float))

    assert torch.isfinite(actual).all()

    torch.testing.assert_close(
        actual,
        torch.tensor(numpy.log(numpy.expm1(x)), dtype=torch.float),
    )
//  [python]
def test_soft_round():
    x = torch.linspace(-2.0, 2.0, 50)

    t.assert_allclose(
        x,
        soft_round(x, alpha=1e-13),
    )

    for offset in range(-5, 5):
        x = torch.linspace(offset - 0.499, offset + 0.499, 100)

        t.assert_allclose(
            torch.round(x),
            soft_round(x, alpha=2000.0),
            atol=0.02,
            rtol=0.02,
        )
//  [python]
def test_soft_round():
    x = torch.linspace(-2.0, 2.0, 50)

    torch.testing.assert_close(
        x,
        soft_round(x, alpha=1e-13),
    )

    for offset in range(-5, 5):
        x = torch.linspace(offset - 0.499, offset + 0.499, 100)

        torch.testing.assert_close(
            torch.round(x),
            soft_round(x, alpha=2000.0),
            atol=0.02,
            rtol=0.02,
        )
//  [python]
def run_worker(args, model_class):
    mod = model_class()
    x = torch.randn(batch_size, d_hid)
    y = torch.randn(batch_size, d_hid)

    pipe = pipeline(
        mod,
        args.chunks,
        example_args=(x, y),
    )

    assert pipe.num_stages == 4

    ref_out = mod(x, y)
    out = pipe(x, y)[0]
    torch.testing.assert_allclose(out, ref_out)
    print(f"equivalence test passed {torch.sum(out)} ref {torch.sum(ref_out)}")

    # Check qualname
    # state_dict.keys include both parameters and persistent buffers
    old_names = set(mod.state_dict().keys())
    new_names = set()
    for idx in range(pipe.num_stages):
        stage_mod = pipe.get_stage_module(idx)
        new_names.update(stage_mod.state_dict().keys())

    assert (
        old_names == new_names
    ), f"""
    old names {old_names}
    new names {new_names}
    """
    print("Qualname check passed")
//  [python]
def run_worker(args, model_class):
    mod = model_class()
    x = torch.randn(batch_size, d_hid)
    y = torch.randn(batch_size, d_hid)

    pipe = pipeline(
        mod,
        args.chunks,
        example_args=(x, y),
    )

    assert pipe.num_stages == 4

    ref_out = mod(x, y)
    out = pipe(x, y)[0]
    torch.testing.assert_close(out, ref_out)
    print(f"equivalence test passed {torch.sum(out)} ref {torch.sum(ref_out)}")

    # Check qualname
    # state_dict.keys include both parameters and persistent buffers
    old_names = set(mod.state_dict().keys())
    new_names = set()
    for idx in range(pipe.num_stages):
        stage_mod = pipe.get_stage_module(idx)
        new_names.update(stage_mod.state_dict().keys())

    assert (
        old_names == new_names
    ), f"""
    old names {old_names}
    new names {new_names}
    """
    print("Qualname check passed")
//  [python]
def _assert_scores_allclose_long_enough(
    first_scores: Sequence[torch.Tensor], second_scores: Sequence[torch.Tensor]
) -> int:
    for i in range(3):
        torch.testing.assert_allclose(
            first_scores[i],
            second_scores[i],
            atol=3e-3,
            rtol=1e-05,
            msg=lambda msg: f"Diverged at {'%d%s' % (i + 1,'tsnrhtdd'[((i + 1)//10%10!=1)*((i + 1)%10<4)*(i + 1)%10::4])} token: {msg}",
        )
//  [python]
def _assert_scores_allclose_long_enough(
    first_scores: Sequence[torch.Tensor], second_scores: Sequence[torch.Tensor]
) -> int:
    for i in range(3):
        torch.testing.assert_close(
            first_scores[i],
            second_scores[i],
            atol=3e-3,
            rtol=1e-05,
            msg=lambda msg: f"Diverged at {'%d%s' % (i + 1,'tsnrhtdd'[((i + 1)//10%10!=1)*((i + 1)%10<4)*(i + 1)%10::4])} token: {msg}",
        )
//  [python]
def test_encoder(model_name, sharded, devices):
    torch.manual_seed(0)

    model = T5ForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True)

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 3), device=devices[0])

    out1_ref = model.get_encoder()(inp1)
    out2_ref = model.get_encoder()(inp2)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    assert isinstance(model_tp, TensorParallelPreTrainedModel)
    del model

    out1 = model_tp.get_encoder()(inp1)
    out2 = model_tp.get_encoder()(inp2)

    torch.testing.assert_allclose(out1_ref, out1, atol=3e-3, rtol=1e-05)
    torch.testing.assert_allclose(out2_ref, out2, atol=3e-3, rtol=1e-05)
//  [python]
def test_encoder(model_name, sharded, devices):
    torch.manual_seed(0)

    model = T5ForConditionalGeneration.from_pretrained(model_name, low_cpu_mem_usage=True)

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 3), device=devices[0])

    out1_ref = model.get_encoder()(inp1)
    out2_ref = model.get_encoder()(inp2)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    assert isinstance(model_tp, TensorParallelPreTrainedModel)
    del model

    out1 = model_tp.get_encoder()(inp1)
    out2 = model_tp.get_encoder()(inp2)

    torch.testing.assert_close(out1_ref, out1, atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out2_ref, out2, atol=3e-3, rtol=1e-05)
//  [python]
def test_forward_bert_like(use_lora, sharded, devices, model_name):
    torch.manual_seed(0)

    model = BertModel.from_pretrained(model_name).to(devices[0])
    if use_lora:
        model = add_lora(model, model_name)

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 1), device=devices[0])
    inp3 = torch.randint(1, 1000, size=(2, 2), device=devices[0])

    out1_ref = model(inp1, output_hidden_states=True)
    out2_ref = model(inp2, output_hidden_states=True)
    out3_ref = model(inp3, output_hidden_states=True)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    del model

    out1 = model_tp(inp1, output_hidden_states=True)
    out2 = model_tp(inp2, output_hidden_states=True)
    out3 = model_tp(inp3, output_hidden_states=True)

    torch.testing.assert_allclose(out1_ref.hidden_states[-1], out1.hidden_states[-1], atol=3e-3, rtol=1e-05)
    torch.testing.assert_allclose(out2_ref.hidden_states[-1], out2.hidden_states[-1], atol=3e-3, rtol=1e-05)
    torch.testing.assert_allclose(out3_ref.hidden_states[-1], out3.hidden_states[-1], atol=3e-3, rtol=1e-05)
//  [python]
def test_forward_bert_like(use_lora, sharded, devices, model_name):
    torch.manual_seed(0)

    model = BertModel.from_pretrained(model_name).to(devices[0])
    if use_lora:
        model = add_lora(model, model_name)

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 1), device=devices[0])
    inp3 = torch.randint(1, 1000, size=(2, 2), device=devices[0])

    out1_ref = model(inp1, output_hidden_states=True)
    out2_ref = model(inp2, output_hidden_states=True)
    out3_ref = model(inp3, output_hidden_states=True)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    del model

    out1 = model_tp(inp1, output_hidden_states=True)
    out2 = model_tp(inp2, output_hidden_states=True)
    out3 = model_tp(inp3, output_hidden_states=True)

    torch.testing.assert_close(out1_ref.hidden_states[-1], out1.hidden_states[-1], atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out2_ref.hidden_states[-1], out2.hidden_states[-1], atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out3_ref.hidden_states[-1], out3.hidden_states[-1], atol=3e-3, rtol=1e-05)
//  [python]
def test_workaround(self):
        model = Model()
        save_model(model, "tmp.safetensors")
        with safe_open("tmp.safetensors", framework="pt") as f:
            self.assertEqual(f.metadata(), {"b.bias": "a.bias", "b.weight": "a.weight"})

        model2 = Model()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.allclose(v, state_dict[k])
//  [python]
def test_workaround(self):
        model = Model()
        save_model(model, "tmp.safetensors")
        with safe_open("tmp.safetensors", framework="pt") as f:
            self.assertEqual(f.metadata(), {"b.bias": "a.bias", "b.weight": "a.weight"})

        model2 = Model()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_close(v, state_dict[k])
//  [python]
def test_forward_gpt2_like(use_lora, sharded, devices, model_name):
    torch.manual_seed(0)

    model = prepare_model(model_name, use_lora).to(devices[0])

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 1), device=devices[0])
    inp3 = torch.randint(1, 1000, size=(2, 2), device=devices[0])

    out1_ref = model(inp1, use_cache=True, output_hidden_states=True)
    out2_ref = model(inp2, use_cache=True, past_key_values=out1_ref.past_key_values)
    out3_ref = model(inp3, use_cache=True, past_key_values=out2_ref.past_key_values)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    del model

    out1 = model_tp(inp1, use_cache=True, output_hidden_states=True)
    out2 = model_tp(inp2, use_cache=True, past_key_values=out1.past_key_values)
    out3 = model_tp(inp3, use_cache=True, past_key_values=out2.past_key_values)

    t.assert_close(out1_ref.hidden_states[-1], out1.hidden_states[-1], atol=3e-3, rtol=1e-05)
    t.assert_close(out1_ref.logits, out1.logits, atol=3e-3, rtol=1e-05)
    t.assert_close(out2_ref.logits, out2.logits, atol=3e-3, rtol=1e-05)
    t.assert_close(out3_ref.logits, out3.logits, atol=3e-3, rtol=1e-05)
//  [python]
def test_forward_gpt2_like(use_lora, sharded, devices, model_name):
    torch.manual_seed(0)

    model = prepare_model(model_name, use_lora).to(devices[0])

    inp1 = torch.randint(1, 1000, size=(2, 3), device=devices[0])
    inp2 = torch.randint(1, 1000, size=(2, 1), device=devices[0])
    inp3 = torch.randint(1, 1000, size=(2, 2), device=devices[0])

    out1_ref = model(inp1, use_cache=True, output_hidden_states=True)
    out2_ref = model(inp2, use_cache=True, past_key_values=out1_ref.past_key_values)
    out3_ref = model(inp3, use_cache=True, past_key_values=out2_ref.past_key_values)

    model_tp = TensorParallelPreTrainedModel(model, devices, sharded=sharded)
    del model

    out1 = model_tp(inp1, use_cache=True, output_hidden_states=True)
    out2 = model_tp(inp2, use_cache=True, past_key_values=out1.past_key_values)
    out3 = model_tp(inp3, use_cache=True, past_key_values=out2.past_key_values)

    torch.testing.assert_close(out1_ref.hidden_states[-1], out1.hidden_states[-1], atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out1_ref.logits, out1.logits, atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out2_ref.logits, out2.logits, atol=3e-3, rtol=1e-05)
    torch.testing.assert_close(out3_ref.logits, out3.logits, atol=3e-3, rtol=1e-05)
//  [python]
def test_workaround_works_with_different_on_file_names(self):
        model = Model()
        state_dict = model.state_dict()
        state_dict.pop("a.weight")
        state_dict.pop("a.bias")
        save_file(state_dict, "tmp.safetensors")

        model2 = Model()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_allclose(v, state_dict[k])
//  [python]
def test_workaround_works_with_different_on_file_names(self):
        model = Model()
        state_dict = model.state_dict()
        state_dict.pop("a.weight")
        state_dict.pop("a.bias")
        save_file(state_dict, "tmp.safetensors")

        model2 = Model()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_close(v, state_dict[k])
//  [python]
def test_workaround_non_contiguous(self):
        model = NonContiguousModel()

        with self.assertRaises(ValueError) as ctx:
            save_model(model, "tmp_c.safetensors", force_contiguous=False)
        self.assertIn("use save_model(..., force_contiguous=True)", str(ctx.exception))
        save_model(model, "tmp_c.safetensors", force_contiguous=True)

        model2 = NonContiguousModel()
        load_model(model2, "tmp_c.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            self.assertEqual(v.shape, state_dict[k].shape)
            self.assertTrue(torch.allclose(v, state_dict[k]))
//  [python]
def test_workaround_non_contiguous(self):
        model = NonContiguousModel()

        with self.assertRaises(ValueError) as ctx:
            save_model(model, "tmp_c.safetensors", force_contiguous=False)
        self.assertIn("use save_model(..., force_contiguous=True)", str(ctx.exception))
        save_model(model, "tmp_c.safetensors", force_contiguous=True)

        model2 = NonContiguousModel()
        load_model(model2, "tmp_c.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_close(v, state_dict[k])
//  [python]
def test_save(self):
        # Just testing the actual saved file to make sure we're ok on big endian
        model = OnesModel()
        save_model(model, "tmp_ones.safetensors")
        with safe_open("tmp_ones.safetensors", framework="pt") as f:
            self.assertEqual(f.metadata(), {"b.bias": "a.bias", "b.weight": "a.weight"})

        # 192 hardcoded to skip the header, metadata order is random.
        self.assertEqual(
            open("tmp_ones.safetensors", "rb").read()[192:],
            b"""\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?""",
        )

        model2 = OnesModel()
        load_model(model2, "tmp_ones.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            self.assertTrue(torch.allclose(v, state_dict[k]))
//  [python]
def test_save(self):
        # Just testing the actual saved file to make sure we're ok on big endian
        model = OnesModel()
        save_model(model, "tmp_ones.safetensors")
        with safe_open("tmp_ones.safetensors", framework="pt") as f:
            self.assertEqual(f.metadata(), {"b.bias": "a.bias", "b.weight": "a.weight"})

        # 192 hardcoded to skip the header, metadata order is random.
        self.assertEqual(
            open("tmp_ones.safetensors", "rb").read()[192:],
            b"""\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?\x00\x00\x80?""",
        )

        model2 = OnesModel()
        load_model(model2, "tmp_ones.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_close(v, state_dict[k])
//  [python]
def test_workaround_copy(self):
        model = CopyModel()
        self.assertEqual(
            _find_shared_tensors(model.state_dict()), [{"a.weight"}, {"a.bias"}, {"b.weight"}, {"b.bias"}]
        )
        save_model(model, "tmp.safetensors")

        model2 = CopyModel()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            self.assertEqual(v, state_dict[k])
//  [python]
def test_workaround_copy(self):
        model = CopyModel()
        self.assertEqual(
            _find_shared_tensors(model.state_dict()), [{"a.weight"}, {"a.bias"}, {"b.weight"}, {"b.bias"}]
        )
        save_model(model, "tmp.safetensors")

        model2 = CopyModel()
        load_model(model2, "tmp.safetensors")

        state_dict = model.state_dict()
        for k, v in model2.state_dict().items():
            torch.testing.assert_close(v, state_dict[k])
//  [python]
def _test_two_configs(
        optim_factory1: Callable[
            [Iterable[Parameter]],
            torch.optim.Optimizer,
        ],
        optim_factory2: Callable[
            [Iterable[Parameter]],
            torch.optim.Optimizer,
        ],
        device: torch.device,
    ) -> None:
        params1, loss1 = ShampooDDPDistributorTest._train_model(
            optim_factory1,
            device=device,
        )
        params2, loss2 = ShampooDDPDistributorTest._train_model(
            optim_factory2,
            device=device,
        )
        t.assert_close(loss1, loss2)
        t.assert_close(params1, params2)
//  [python]
def _test_two_configs(
        optim_factory1: Callable[
            [Iterable[Parameter]],
            torch.optim.Optimizer,
        ],
        optim_factory2: Callable[
            [Iterable[Parameter]],
            torch.optim.Optimizer,
        ],
        device: torch.device,
    ) -> None:
        params1, loss1 = ShampooDDPDistributorTest._train_model(
            optim_factory1,
            device=device,
        )
        params2, loss2 = ShampooDDPDistributorTest._train_model(
            optim_factory2,
            device=device,
        )
        torch.testing.assert_close(loss1, loss2)
        torch.testing.assert_close(params1, params2)
//  [python]
def _test_argmax(
        self,
        batch_size=1,
        shape=(2, 6),
        dim=0,
        test_name="argmax",
        copy_op=False,
        dtype="float16",
    ):
        o_shape = list(shape)[:-1]

        X1 = Tensor(
            shape=shape,
            dtype=dtype,
            name="X",
            is_input=True,
        )
        X4_op = ops.argmax(dim=dim)
        X4 = X4_op(X1)
        X4._attrs["is_output"] = True
        X4._attrs["name"] = "output"

        target = detect_target()
        module = compile_model(X4, target, "./tmp", test_name)

        scores = get_random_torch_tensor(shape, dtype=dtype)
        y_pt = torch.argmax(scores, dim=dim)
        y = torch.empty_like(y_pt, dtype=torch.int64)

        module.run_with_tensors([scores], [y])
        y_reshape = y.reshape(o_shape)
        torch.testing.assert_allclose(y_pt, y_reshape, atol=0, rtol=0)
//  [python]
def _test_argmax(
        self,
        batch_size=1,
        shape=(2, 6),
        dim=0,
        test_name="argmax",
        copy_op=False,
        dtype="float16",
    ):
        o_shape = list(shape)[:-1]

        X1 = Tensor(
            shape=shape,
            dtype=dtype,
            name="X",
            is_input=True,
        )
        X4_op = ops.argmax(dim=dim)
        X4 = X4_op(X1)
        X4._attrs["is_output"] = True
        X4._attrs["name"] = "output"

        target = detect_target()
        module = compile_model(X4, target, "./tmp", test_name)

        scores = get_random_torch_tensor(shape, dtype=dtype)
        y_pt = torch.argmax(scores, dim=dim)
        y = torch.empty_like(y_pt, dtype=torch.int64)

        module.run_with_tensors([scores], [y])
        y_reshape = y.reshape(o_shape)
        torch.testing.assert_close(y_pt, y_reshape, atol=0, rtol=0)
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        for batch_size in [1, 4]:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for seq_length in [16, 89, 128]:
                dataset = deepcopy(self.eval_dataset)
                dataset.set_transform(partial(self.data_transform, max_length=seq_length))
                for inputs in self.trainer.get_eval_dataloader(dataset):
                    self.assertSequenceEqual(inputs["input_ids"].shape, [batch_size, seq_length])
                    ovmodel_outputs = ovmodel(**inputs)
                    self.assertIn("logits", ovmodel_outputs)
                    ovmodel_logits = ovmodel_outputs.logits
                    with torch.no_grad():
                        torch_logits = torch_model(**inputs).logits
                    torch.testing.assert_allclose(
                        ovmodel_logits,
                        torch_logits,
                        atol=1e-3,
                        rtol=1e-4,
                    )
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        for batch_size in [1, 4]:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for seq_length in [16, 89, 128]:
                dataset = deepcopy(self.eval_dataset)
                dataset.set_transform(partial(self.data_transform, max_length=seq_length))
                for inputs in self.trainer.get_eval_dataloader(dataset):
                    self.assertSequenceEqual(inputs["input_ids"].shape, [batch_size, seq_length])
                    ovmodel_outputs = ovmodel(**inputs)
                    self.assertIn("logits", ovmodel_outputs)
                    ovmodel_logits = ovmodel_outputs.logits
                    with torch.no_grad():
                        torch_logits = torch_model(**inputs).logits
                    torch.testing.assert_close(
                        ovmodel_logits,
                        torch_logits,
                        atol=1e-3,
                        rtol=1e-4,
                    )
//  [python]
def test_difference_torch_odd(self):
        model = NoSharedModel()
        a = model.a.weight
        b = model.b.weight
        self.assertNotEqual(a.data_ptr(), b.data_ptr())
        torch.save(model.state_dict(), "tmp3.bin")

        model2 = Model()
        self.assertEqual(_find_shared_tensors(model2.state_dict()), [{"a.weight", "b.weight"}, {"b.bias", "a.bias"}])
        # Torch will affect either `b` or `a` to the shared tensor in the `model2`
        model2.load_state_dict(torch.load("tmp3.bin"))

        # XXX: model2 uses only the B weight not the A weight anymore.
        self.assertFalse(torch.allclose(model2.a.weight, model.a.weight))
        self.assertTrue(torch.allclose(model2.a.weight, model.b.weight))
        self.assertEqual(_find_shared_tensors(model2.state_dict()), [{"a.weight", "b.weight"}, {"b.bias", "a.bias"}])

        # Everything is saved as-is
        save_model(model, "tmp3.safetensors")
        # safetensors will yell that there were 2 tensors on disk, while
        # the models expects only 1 tensor since both are shared.
        with self.assertRaises(RuntimeError) as ctx:
            load_model(model2, "tmp3.safetensors")
        # Safetensors properly warns the user that some ke
        self.assertIn("""Unexpected key(s) in state_dict: "b.bias", "b.weight""", str(ctx.exception))
//  [python]
def test_difference_torch_odd(self):
        model = NoSharedModel()
        a = model.a.weight
        b = model.b.weight
        self.assertNotEqual(a.data_ptr(), b.data_ptr())
        torch.save(model.state_dict(), "tmp3.bin")

        model2 = Model()
        self.assertEqual(_find_shared_tensors(model2.state_dict()), [{"a.weight", "b.weight"}, {"b.bias", "a.bias"}])
        # Torch will affect either `b` or `a` to the shared tensor in the `model2`
        model2.load_state_dict(torch.load("tmp3.bin"))

        # XXX: model2 uses only the B weight not the A weight anymore.
        self.assertFalse(torch.allclose(model2.a.weight, model.a.weight))
        torch.testing.assert_close(model2.a.weight, model.b.weight)
        self.assertEqual(_find_shared_tensors(model2.state_dict()), [{"a.weight", "b.weight"}, {"b.bias", "a.bias"}])

        # Everything is saved as-is
        save_model(model, "tmp3.safetensors")
        # safetensors will yell that there were 2 tensors on disk, while
        # the models expects only 1 tensor since both are shared.
        with self.assertRaises(RuntimeError) as ctx:
            load_model(model2, "tmp3.safetensors")
        # Safetensors properly warns the user that some ke
        self.assertIn("""Unexpected key(s) in state_dict: "b.bias", "b.weight""", str(ctx.exception))
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        batch_sizes = [1] if self.is_swin else [1, 4]
        for batch_size in batch_sizes:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for inputs in self.trainer.get_eval_dataloader():
                self.assertEqual(inputs["pixel_values"].shape[0], batch_size)
                ovmodel_outputs = ovmodel(**inputs)
                self.assertIn("logits", ovmodel_outputs)
                ovmodel_logits = ovmodel_outputs.logits
                with torch.no_grad():
                    torch_logits = torch_model(**inputs).logits
                self.assertTrue(torch.allclose(ovmodel_logits, torch_logits, atol=1e-3, rtol=1e-4))
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        batch_sizes = [1] if self.is_swin else [1, 4]
        for batch_size in batch_sizes:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for inputs in self.trainer.get_eval_dataloader():
                self.assertEqual(inputs["pixel_values"].shape[0], batch_size)
                ovmodel_outputs = ovmodel(**inputs)
                self.assertIn("logits", ovmodel_outputs)
                ovmodel_logits = ovmodel_outputs.logits
                with torch.no_grad():
                    torch_logits = torch_model(**inputs).logits
                torch.testing.assert_close(
                    ovmodel_logits,
                    torch_logits,
                    atol=1e-3,
                    rtol=1e-4,
                )
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        for batch_size in [1, 4]:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for seq_length in [12345, 16000]:
                dataset = deepcopy(self.eval_dataset)
                dataset.set_transform(partial(self.data_transform, max_length=seq_length))
                for inputs in self.trainer.get_eval_dataloader(dataset):
                    self.assertSequenceEqual(inputs["input_values"].shape, [batch_size, seq_length])
                    ovmodel_outputs = ovmodel(**inputs)
                    self.assertIn("logits", ovmodel_outputs)
                    ovmodel_logits = ovmodel_outputs.logits
                    with torch.no_grad():
                        torch_logits = torch_model(**inputs).logits
                    torch.testing.assert_allclose(
                        ovmodel_logits,
                        torch_logits,
                        atol=1e-3,
                        rtol=1e-4,
                    )
//  [python]
def check_ovmodel_output_equals_torch_output(self, ovmodel, torch_model):
        torch_model = torch_model.eval()
        for batch_size in [1, 4]:
            self.trainer.args = self.get_training_args(eval_batch_size=batch_size)
            self.trainer.create_accelerator_and_postprocess()
            for seq_length in [12345, 16000]:
                dataset = deepcopy(self.eval_dataset)
                dataset.set_transform(partial(self.data_transform, max_length=seq_length))
                for inputs in self.trainer.get_eval_dataloader(dataset):
                    self.assertSequenceEqual(inputs["input_values"].shape, [batch_size, seq_length])
                    ovmodel_outputs = ovmodel(**inputs)
                    self.assertIn("logits", ovmodel_outputs)
                    ovmodel_logits = ovmodel_outputs.logits
                    with torch.no_grad():
                        torch_logits = torch_model(**inputs).logits
                    torch.testing.assert_close(
                        ovmodel_logits,
                        torch_logits,
                        atol=1e-3,
                        rtol=1e-4,
                    )
//  [python]
def test_word_information_preserved_with_valid_input(self) -> None:
        t.assert_allclose(
            word_information_preserved("hello meta", "hi metaverse"),
            torch.tensor(0.0, dtype=torch.float64),
        )
        t.assert_allclose(
            word_information_preserved("hello meta", "hello meta"),
            torch.tensor(1.0, dtype=torch.float64),
        )
        t.assert_allclose(
            word_information_preserved(
                "this is the prediction", "this is the reference"
            ),
            torch.tensor(0.5625, dtype=torch.float64),
        )
        t.assert_allclose(
            word_information_preserved(
                ["hello world", "welcome to the facebook"],
                ["hello metaverse", "welcome to meta"],
            ),
            torch.tensor(0.3, dtype=torch.float64),
        )
        t.assert_allclose(
            word_information_preserved(
                [
                    "hello metaverse",
                    "come to the facebook",
                    "this is reference",
                    "there is the other one",
                ],
                [
                    "hello world",
                    "welcome to meta",
                    "this is reference",
                    "there is another one",
                ],
            ),
            torch.tensor(0.38095238, dtype=torch.float64),
        )
//  [python]
def test_word_information_preserved_with_valid_input(self) -> None:
        torch.testing.assert_close(
            word_information_preserved("hello meta", "hi metaverse"),
            torch.tensor(0.0, dtype=torch.float64),
        )
        torch.testing.assert_close(
            word_information_preserved("hello meta", "hello meta"),
            torch.tensor(1.0, dtype=torch.float64),
        )
        torch.testing.assert_close(
            word_information_preserved(
                "this is the prediction", "this is the reference"
            ),
            torch.tensor(0.5625, dtype=torch.float64),
        )
        torch.testing.assert_close(
            word_information_preserved(
                ["hello world", "welcome to the facebook"],
                ["hello metaverse", "welcome to meta"],
            ),
            torch.tensor(0.3, dtype=torch.float64),
        )
        torch.testing.assert_close(
            word_information_preserved(
                [
                    "hello metaverse",
                    "come to the facebook",
                    "this is reference",
                    "there is the other one",
                ],
                [
                    "hello world",
                    "welcome to meta",
                    "this is reference",
                    "there is another one",
                ],
            ),
            torch.tensor(0.38095238, dtype=torch.float64),
        )
//  [python]
def test_single_step_vs_multi(self, shape, t, python_based):
        td = TensorDict(
            {
                "observation": torch.arange(t, dtype=torch.float32)
                .unsqueeze(-1)
                .expand(*shape, t, 3),
                "is_init": torch.zeros(*shape, t, 1, dtype=torch.bool),
            },
            [*shape, t],
        )
        lstm_module_ss = LSTMModule(
            input_size=3,
            hidden_size=12,
            batch_first=True,
            in_keys=["observation", "hidden0", "hidden1"],
            out_keys=["intermediate", ("next", "hidden0"), ("next", "hidden1")],
            python_based=python_based,
        )
        lstm_module_ms = lstm_module_ss.set_recurrent_mode()
        lstm_module_ms(td)
        td_ss = TensorDict(
            {
                "observation": torch.zeros(*shape, 3),
                "is_init": torch.zeros(*shape, 1, dtype=torch.bool),
            },
            shape,
        )
        for _t in range(t):
            lstm_module_ss(td_ss)
            td_ss = step_mdp(td_ss, keep_other=True)
            td_ss["observation"][:] = _t + 1
        torch.testing.assert_allclose(
            td_ss["hidden0"], td["next", "hidden0"][..., -1, :, :]
        )
//  [python]
def test_single_step_vs_multi(self, shape, t, python_based):
        td = TensorDict(
            {
                "observation": torch.arange(t, dtype=torch.float32)
                .unsqueeze(-1)
                .expand(*shape, t, 3),
                "is_init": torch.zeros(*shape, t, 1, dtype=torch.bool),
            },
            [*shape, t],
        )
        lstm_module_ss = LSTMModule(
            input_size=3,
            hidden_size=12,
            batch_first=True,
            in_keys=["observation", "hidden0", "hidden1"],
            out_keys=["intermediate", ("next", "hidden0"), ("next", "hidden1")],
            python_based=python_based,
        )
        lstm_module_ms = lstm_module_ss.set_recurrent_mode()
        lstm_module_ms(td)
        td_ss = TensorDict(
            {
                "observation": torch.zeros(*shape, 3),
                "is_init": torch.zeros(*shape, 1, dtype=torch.bool),
            },
            shape,
        )
        for _t in range(t):
            lstm_module_ss(td_ss)
            td_ss = step_mdp(td_ss, keep_other=True)
            td_ss["observation"][:] = _t + 1
        torch.testing.assert_close(
            td_ss["hidden0"], td["next", "hidden0"][..., -1, :, :]
        )
//  [python]
def test_logn_attention(self):
        torch.classes.load_library(os.environ['TEST_SRCDIR'] + "/maga_transformer/tests/libtest_ops.so")
        
        for logn_seq_len in range(1024, 2048, 512):
            self.LognAttentionOp = torch.classes.unittest.LognAttentionOp(logn_seq_len)
            for len in range(1024, 8096, 1024):
                data = torch.rand([32, len, 2, 128]).cuda()
                result = self.LognAttentionOp.forward(data)
                test = logn_attention_test(data, logn_seq_len)
                torch.testing.assert_allclose(test, result, rtol=1e-2, atol=1e-3)
//  [python]
def test_logn_attention(self):
        torch.classes.load_library(os.environ['TEST_SRCDIR'] + "/maga_transformer/tests/libtest_ops.so")
        
        for logn_seq_len in range(1024, 2048, 512):
            self.LognAttentionOp = torch.classes.unittest.LognAttentionOp(logn_seq_len)
            for len in range(1024, 8096, 1024):
                data = torch.rand([32, len, 2, 128]).cuda()
                result = self.LognAttentionOp.forward(data)
                test = logn_attention_test(data, logn_seq_len)
                torch.testing.assert_close(test, result, rtol=1e-2, atol=1e-3)
//  [python]
def test_video_transforms(self):
        config = OmegaConf.create(
            {
                "transforms": [
                    "permute_and_rescale",
                    {"type": "Resize", "params": {"size": [140, 100]}},
                    {"type": "UniformTemporalSubsample", "params": {"num_samples": 7}},
                ]
            }
        )
        video_transforms = VideoTransforms(config)

        video = torch.ones(10, 200, 200, 3)
        processed_video = video_transforms(video)
        torch.allclose(processed_video, torch.ones(3, 7, 140, 100) / 255)
//  [python]
def test_video_transforms(self):
        config = OmegaConf.create(
            {
                "transforms": [
                    "permute_and_rescale",
                    {"type": "Resize", "params": {"size": [140, 100]}},
                    {"type": "UniformTemporalSubsample", "params": {"num_samples": 7}},
                ]
            }
        )
        video_transforms = VideoTransforms(config)

        video = torch.ones(10, 200, 200, 3)
        processed_video = video_transforms(video)
        torch.testing.assert_close(processed_video, torch.ones(3, 7, 140, 100) / 255)