Easier perplexity computation
Author: uditarora
Pull request: no
Labels: name: Feature request

# 🚀 Feature request

The docs provide a method to evaluate perplexity for a GPT-2 model, one example at a time (https://huggingface.co/transformers/perplexity.html). However this can potentially be included in the library with the computation being done in a batched manner. 

## Motivation

This would make it easier and faster for people to evaluate their language models in terms of perplexity.

If not a solution integrated in the library, the example given in the docs can be updated to do computation in a batched manner for speed.


Comments:
Hi @uditarora! That would be a nice addition, do you want to open a PR to add a batched computation to the documentation?

Sure! I can try to create one.

Might take me a couple of weeks before I can get started on it though, due to prior commitments.

Hi all. Is this issue still open? I like to contribute and collaborate.

Here's my take.

```python3
import torch
import torch.nn.functional as F
from tqdm import tqdm
from transformers import GPT2LMHeadModel, GPT2TokenizerFast
from datasets import load_dataset


def batched_perplexity(model, dataset, tokenizer, batch_size, stride):
    device = model.device
    encodings = tokenizer("\n\n".join(dataset["text"]), return_tensors="pt")
    text_len = encodings.input_ids.size(1)
    lls = []

    for i in tqdm(range(0, text_len, batch_size * stride)):
        begin_locs, end_locs, trg_lens = [], [], []
        for j in range(batch_size):
            j = i + j * stride
            if j >= text_len:
                break
            begin_loc = max(j + stride - max_len, 0)
            end_loc = min(j + stride, text_len)
            trg_len = end_loc - j  # may be different from stride on last loop

            begin_locs.append(begin_loc)
            end_locs.append(end_loc)
            trg_lens.append(trg_len)

        input_ids = [encodings.input_ids[:, b:e] for b, e in zip(begin_locs, end_locs)]
        target_end_locs = [sen.size(-1) for sen in input_ids]
        input_ids = [
            F.pad(sen, (0, max_len - sen.size(-1)), "constant", 0) for sen in input_ids
        ] # we dont need attention mask as long as these padded token is not involved in loss calculation
        input_ids = torch.stack(input_ids, dim=1).squeeze(0).to(device)

        target_ids = torch.ones_like(input_ids) * -100 # -100 is the default ingore_index value in torch.nn.CrossEntropyLoss
        for i, (b, e) in enumerate(zip(trg_lens, target_end_locs)):
            labels = input_ids[i, -b:e].clone()
            target_ids[i, -b:e] = labels

        with torch.no_grad():
            outputs = model(input_ids, labels=target_ids)
            log_likelihood = outputs["loss"] * sum(trg_lens)

        lls.append(log_likelihood)

    ppl = torch.exp(sum(torch.stack(lls) / end_locs[-1]))
    return ppl


if __name__ == "__main__":
    device = "cuda"
    model_id = "distilgpt2"
    model = GPT2LMHeadModel.from_pretrained(model_id).to(device)
    tokenizer = GPT2TokenizerFast.from_pretrained(model_id)
    max_len = model.config.n_positions
    stride = 512
    batch_size = 16

    test = load_dataset("wikitext", "wikitext-2-raw-v1", split="test[:100%]")
    ppl = batched_perplexity(model, test, tokenizer, batch_size, stride)
    print(f"--------------{ppl=}-------------")
```

Hello! is this issue still open? Did you test that the batched example above gives the same value for GPT2 as the documentation?

I hope this is correct? 
```
model = GPT2LMHeadModel.from_pretrained("/content/drive/MyDrive/Colab Notebooks/best_model_wiki")
def calculate_perplexity(model, dataloader, device):
    model.eval()  # make sure the model is in evaluation mode
    total_loss = 0
    total_examples = 0

    with torch.no_grad():  # we don't need to calculate gradients
        for batch in dataloader:
            inputs, labels = batch['input_ids'].to(device), batch['labels'].to(device)
            outputs = model(input_ids=inputs, labels=labels)
            total_loss += outputs.loss.item() * inputs.size(0)
            total_examples += inputs.size(0)

    average_loss = total_loss / total_examples
    perplexity = torch.exp(torch.tensor(average_loss))
    return perplexity
from torch.utils.data import DataLoader

# Assuming you have a GPU available
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Create the DataLoader for our test dataset
test_dataloader = DataLoader(test_dataset, batch_size=4, collate_fn=data_collator)

# Calculate the perplexity
test_perplexity = calculate_perplexity(model, test_dataloader, device)
print(f"Test Perplexity: {test_perplexity}")
```