How do I calculate the `decoder_input_ids` for calling the model?
#2
by
basrah
- opened
I would like to get scores/token probabilities from this model in the toxic chat example, to evaluate the precision, recall and so on.
model.forward seems to need decoder_intput_ids, how do I calculate this?
Any help is much appreciated :)
Hi, this is how you could calculate the probabilities for positive/negative.
NEAGTIVE_TOKEN_ID = 2841
POSITIVE_TOKEN_ID = 1465
prediction_scores = []
for step, batch in tqdm(enumerate(eval_dataloader), total=len(eval_dataloader)):
batch = {k: v.to(device) for k, v in batch.items()}
with torch.no_grad():
outputs = model.generate(
batch["input_ids"],
max_new_tokens=max_target_length,
attention_mask=batch["attention_mask"],
output_scores=True,
return_dict_in_generate=True
)
sequences_negative = torch.tensor(
[[0, NEAGTIVE_TOKEN_ID, 1] for _ in range(outputs.sequences.shape[0])]
).to(outputs.sequences.device)
sequences_positive = torch.tensor(
[[0, POSITIVE_TOKEN_ID, 1] for _ in range(outputs.sequences.shape[0])]
).to(outputs.sequences.device)
transition_scores_negative = model.compute_transition_scores(
sequences_negative, outputs.scores, normalize_logits=True
)
transition_scores_positive = model.compute_transition_scores(
sequences_positive, outputs.scores, normalize_logits=True
)
generated_tokens = outputs.sequences
decoded_preds = tokenizer.batch_decode(
generated_tokens, skip_special_tokens=True)
one_hot_preds = [transfer_text_to_label(text) for text in decoded_preds]
labels = batch["labels"]
decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
one_hot_labels = [transfer_text_to_label(text) for text in decoded_labels]
negative_prediction_scores = [np.exp(x[0].cpu().numpy()) for x in transition_scores_negative]
positive_prediction_scores = [np.exp(x[0].cpu().numpy()) for x in transition_scores_positive]
prediction_score_normalized = [y/(x+y) for x, y in zip(negative_prediction_scores, positive_prediction_scores)]
prediction_scores.extend(prediction_score_normalized)