How do I calculate the `decoder_input_ids` for calling the model?

#2
by basrah - opened

I would like to get scores/token probabilities from this model in the toxic chat example, to evaluate the precision, recall and so on.

model.forward seems to need decoder_intput_ids, how do I calculate this?

Any help is much appreciated :)

Hi, this is how you could calculate the probabilities for positive/negative.

NEAGTIVE_TOKEN_ID = 2841
POSITIVE_TOKEN_ID = 1465
prediction_scores = []
for step, batch in tqdm(enumerate(eval_dataloader), total=len(eval_dataloader)):
    batch = {k: v.to(device) for k, v in batch.items()}
    with torch.no_grad():
        outputs = model.generate(
            batch["input_ids"],
            max_new_tokens=max_target_length,
            attention_mask=batch["attention_mask"],
            output_scores=True,
            return_dict_in_generate=True
        )

    sequences_negative = torch.tensor(
        [[0, NEAGTIVE_TOKEN_ID, 1] for _ in range(outputs.sequences.shape[0])]
    ).to(outputs.sequences.device)
    sequences_positive = torch.tensor(
        [[0, POSITIVE_TOKEN_ID, 1] for _ in range(outputs.sequences.shape[0])]
    ).to(outputs.sequences.device)

    transition_scores_negative = model.compute_transition_scores(
        sequences_negative, outputs.scores, normalize_logits=True
    )
    transition_scores_positive = model.compute_transition_scores(
        sequences_positive, outputs.scores, normalize_logits=True
    )

    generated_tokens = outputs.sequences

    decoded_preds = tokenizer.batch_decode(
        generated_tokens, skip_special_tokens=True)
    one_hot_preds = [transfer_text_to_label(text) for text in decoded_preds]

    labels = batch["labels"]
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
    one_hot_labels = [transfer_text_to_label(text) for text in decoded_labels]
    negative_prediction_scores = [np.exp(x[0].cpu().numpy()) for x in transition_scores_negative]
    positive_prediction_scores = [np.exp(x[0].cpu().numpy()) for x in transition_scores_positive]
    prediction_score_normalized = [y/(x+y) for x, y in zip(negative_prediction_scores, positive_prediction_scores)]
    prediction_scores.extend(prediction_score_normalized)

Sign up or log in to comment