Repository: flairNLP/flair

Issue:
Get perplexity of a sentence.
Is it currently possible to get the perplexity (or probability) of a sentence using flair's language model. 


Discussion:
Hello @bedapudi6788 we currently calculate perplexity as part of the language model training, but
there is currently no convenience method for doing this for a sentence.

This would be a good feature to add - so I am adding a "feature" tag to this issue.  


Hi @alanakbik can you point me to the part of code where perplexity is calculated. I might be able to port it. 
Hello @bedapudi6788 it is here: 
 https://github.com/zalandoresearch/flair/blob/4b3562eed534e4e305a36d2f22d12961bc6c25d5/flair/trainers/language_model_trainer.py#L322

Here, we use math.exp() over the cross_entropy loss to get perplexity. 
@alanakbik got it. Can you give me any pointers on calculating perplexity for a single sentence. I will try to implement that.
Great!
Hi @alanakbik , I am using the following method to calculate probability of a sentence.

```
txt = 'i am batman'

model = FlairEmbeddings('news-forward').lm

char_tensors = []
for char in '\n' + txt:
    char_tensors.append(
        torch.tensor(model.dictionary.get_idx_for_item(char)).unsqueeze(0).unsqueeze(0)
    )

log_probs = []

for i in range(len(char_tensors) - 1):
    current_char = txt[i]
    
    input = torch.cat(char_tensors[:i+1])
    hidden = model.init_hidden(1)
    prediction, _, hidden = model.forward(input, hidden)
    prediction = prediction.squeeze().detach()
    if i > 0:
        prediction = prediction[i]

    log_prob = prediction.data[model.dictionary.get_idx_for_item(current_char)].item()
    print(log_prob)
    log_probs.append(log_prob)

print(log_probs)
print(sum(log_probs)/ len(txt))
```

Any thoughts on this?
Hello @bedapudi6788 yes this is one way to get an estimation of how well the sentence fits the model. 

If you want to compute the perplexity though, you need to calculate and exponentiate the cross entropy loss. I think you can do this with this snippet: 

```python
import math
import torch
from flair.embeddings import FlairEmbeddings

# get language model
model = FlairEmbeddings('news-forward').lm

# example text
text = 'The company reported profits.'

# input ids
input = torch.tensor([model.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)

# push list of character IDs through model
hidden = model.init_hidden(1)
prediction, _, hidden = model.forward(input, hidden)

# the target is always the next character
targets = torch.tensor([model.dictionary.get_idx_for_item(char) for char in text[1:]])

# use cross entropy loss to compare output of forward pass with targets
cross_entroy_loss = torch.nn.CrossEntropyLoss()
loss = cross_entroy_loss(prediction.view(-1, len(model.dictionary)), targets).item()

# exponentiate cross-entropy loss to calculate perplexity
perplexity = math.exp(loss)

print(perplexity)
```

@alanakbik I've one question. I tried to emulate the evaluation steps that are done during language model training.

I used the following code:

```python
import math
from pathlib import Path

import torch

import flair
from flair.embeddings import FlairEmbeddings
from flair.trainers.language_model_trainer import TextCorpus, TextDataset

model = FlairEmbeddings("news-forward").lm

class PseudoTextCorpus:
    def __init__(self,
                 forward: bool = True,
                 character_level: bool = True,
                 random_case_flip: bool = True,
                 shuffle_lines: bool = True):
        self.forward = forward
        self.split_on_char = character_level
        self.random_case_flip = random_case_flip
        self.shuffle_lines = shuffle_lines
        self.valid = TextDataset(Path('valid_smal.txt'), model.dictionary, False, self.forward, self.split_on_char,
                                 self.random_case_flip, shuffle_lines=False)[0]

class PerplexityCalculator:

    def __init__(self, model: FlairEmbeddings,
                 corpus: TextCorpus,
                 mini_batch_size = 1,
                 sequence_length = 250):
        self.model = model
        self.corpus = corpus
        self.mini_batch_size = mini_batch_size
        self.sequence_length = sequence_length
        self.loss_function = torch.nn.CrossEntropyLoss()

    @staticmethod
    def _batchify(data, batch_size):
        # Work out how cleanly we can divide the dataset into bsz parts.
        nbatch = data.size(0) // batch_size
        # Trim off any extra elements that wouldn't cleanly fit (remainders).
        data = data.narrow(0, 0, nbatch * batch_size)
        # Evenly divide the data across the bsz batches.
        data = data.view(batch_size, -1).t().contiguous()
        return data

    @staticmethod
    def _get_batch(source, i, sequence_length):
        seq_len = min(sequence_length, len(source) - 1 - i)

        data = source[i:i + seq_len].clone().detach()
        target = source[i + 1:i + 1 + seq_len].view(-1).clone().detach()

        data = data.to(flair.device)
        target = target.to(flair.device)


        return data, target

    @staticmethod
    def _repackage_hidden(h):
        """Wraps hidden states in new tensors, to detach them from their history."""
        return tuple(v.clone().detach() for v in h)

    def _evaluate(self, data_source, eval_batch_size, sequence_length):
        # Turn on evaluation mode which disables dropout.
        self.model.eval()

        with torch.no_grad():
            total_loss = 0
            ntokens = len(self.model.dictionary)

            hidden = self.model.init_hidden(eval_batch_size)

            for i in range(0, data_source.size(0) - 1, sequence_length):
                data, targets = self._get_batch(data_source, i, sequence_length)
                prediction, rnn_output, hidden = self.model.forward(data, hidden)
                output_flat = prediction.view(-1, ntokens)

                total_loss += len(data) * self.loss_function(output_flat, targets).data

                hidden = self._repackage_hidden(hidden)

        return total_loss.item() / len(data_source)

    def calculate_perplexity(self):
        val_data = self._batchify(self.corpus.valid, self.mini_batch_size)
        val_loss = self._evaluate(val_data, self.mini_batch_size, self.sequence_length)

        return math.exp(val_loss)


corpus = PseudoTextCorpus()

ppl_calc = PerplexityCalculator(model=model, corpus=corpus)

print(f"PPL is {ppl_calc.calculate_perplexity()}")
```

I wrote the sentence `The company reported profits.` into the `valid_smal.txt` file. Here's a ppl comparison:

| Code | ppl
| ------ | -----
| `lm training emulation` | *2.9513113840281315*
| Code mentioned here | *3.067617943359323*

Do you know where the discrepancy (0.1163) comes from 🤔
Hi @stefan-it - I think the difference is here: 

```python
                total_loss += len(data) * self.loss_function(output_flat, targets).data

                hidden = self._repackage_hidden(hidden)

        return total_loss.item() / len(data_source)
```

In the `PerplexityCalculator`, the output of the loss function is multiplied by the number of characters (`len(data)`) and then divided by the `len(data_source)`. For the example sentence, `len(data)` is 28, but `len(data_source)` is 29, which is probably wrong (?).

In my code snipped above, this multiplication and division does not happen. It just takes the exponent of the loss directly. 


Resolving patch:
diff --git a/flair/models/language_model.py b/flair/models/language_model.py
--- a/flair/models/language_model.py
+++ b/flair/models/language_model.py
@@ -298,3 +298,27 @@ def generate_text(self, prefix: str = '\n', number_of_characters: int = 1000, te
                 text = text[::-1]
 
             return text, log_prob
+
+    def calculate_perplexity(self, text: str) -> float:
+
+        if not self.is_forward_lm:
+            text = text[::-1]
+
+        # input ids
+        input = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[:-1]]).unsqueeze(1)
+
+        # push list of character IDs through model
+        hidden = self.init_hidden(1)
+        prediction, _, hidden = self.forward(input, hidden)
+
+        # the target is always the next character
+        targets = torch.tensor([self.dictionary.get_idx_for_item(char) for char in text[1:]])
+
+        # use cross entropy loss to compare output of forward pass with targets
+        cross_entroy_loss = torch.nn.CrossEntropyLoss()
+        loss = cross_entroy_loss(prediction.view(-1, len(self.dictionary)), targets).item()
+
+        # exponentiate cross-entropy loss to calculate perplexity
+        perplexity = math.exp(loss)
+
+        return perplexity


Test patch:
diff --git a/tests/test_language_model.py b/tests/test_language_model.py
--- a/tests/test_language_model.py
+++ b/tests/test_language_model.py
@@ -29,3 +29,34 @@ def test_generate_text_with_small_temperatures():
     text, likelihood = language_model.generate_text(temperature=0.01, number_of_characters=100)
     assert (text is not None)
     assert (len(text) >= 100)
+
+
+def test_compute_perplexity():
+
+    from flair.embeddings import FlairEmbeddings
+
+    language_model = FlairEmbeddings('news-forward-fast').lm
+
+    grammatical = 'The company made a profit'
+    perplexity_gramamtical_sentence = language_model.calculate_perplexity(grammatical)
+
+    ungrammatical = 'Nook negh qapla!'
+    perplexity_ungramamtical_sentence = language_model.calculate_perplexity(ungrammatical)
+
+    print(f'"{grammatical}" - perplexity is {perplexity_gramamtical_sentence}')
+    print(f'"{ungrammatical}" - perplexity is {perplexity_ungramamtical_sentence}')
+
+    assert (perplexity_gramamtical_sentence < perplexity_ungramamtical_sentence)
+
+    language_model = FlairEmbeddings('news-backward-fast').lm
+
+    grammatical = 'The company made a profit'
+    perplexity_gramamtical_sentence = language_model.calculate_perplexity(grammatical)
+
+    ungrammatical = 'Nook negh qapla!'
+    perplexity_ungramamtical_sentence = language_model.calculate_perplexity(ungrammatical)
+
+    print(f'"{grammatical}" - perplexity is {perplexity_gramamtical_sentence}')
+    print(f'"{ungrammatical}" - perplexity is {perplexity_ungramamtical_sentence}')
+
+    assert (perplexity_gramamtical_sentence < perplexity_ungramamtical_sentence)