Repository: facebookresearch/ParlAI

Issue:
Could eval_model compute the distinct metrics?
Hi, I would like to compute **the distinct metrics** using the `eval_model` command. 

But I saw the CLI Arguments (as follows), it likely could not support to compute **the distinct metrics**. Is it?

```
-mcs, --metrics | List of metrics to show/compute, e.g. all, default, or give a list split by, like ppl,f1, accuracy, hits@1, rouge, bleu. the rouge metrics will be computed as rouge-1, rouge-2, and rouge-l
```





Discussion:
We don't have any implementation of that right now. I would welcome a PR implementing it.

Implementation would need to go here:
https://github.com/facebookresearch/ParlAI/blob/fb44a721cd5288fa097622e3d0b95e3379f29e74/parlai/core/metrics.py#L719-L735
What's the definition of distinct? Unique n-grams over generations from the entire corpus, or unique n-grams per utterance?
The distinct metric was first introduced in [A Diversity-Promoting Objective Function for Neural Conversation Models](https://arxiv.org/pdf/1510.03055.pdf), the original paper said as follows

> We report the degree of diversity by calculating the number of distinct unigrams and bigrams in generated responses. The value is scaled by the total number of generated tokens to avoid favoring long sentences (shown as distinct-1 and distinct-2 in Tables 2 and 3).

But they didn't release their implementation, we usually compute the n-grams over generations from the entire corpus (it will be very higher than the original paper if computed per utterance)

[Here](https://github.com/PaddlePaddle/models/blob/release/1.6/PaddleNLP/Research/Dialogue-PLATO/plato/metrics/metrics.py) is an implementation. I will add it to ParlAI later :(
Hm, the method I linked you to assumes metrics are computed per-utterance, independently across the corpus. Computing IntraDistinct should be relatively straightforward. InterDistinct is a bit trickier, as we need to keep the ngrams across the corpus. Something like this should work, at least when not in distributed mode.


```python
class InterDistinctMetric(Metric):
    def __init__(self, counts):
        self._counts = counts

    def __add__(self, other):
        return DistinctMetric(self._counts + other._counts)

    def value(self):
        return (len(self._counts) + 1e-12) / (sum(self._counts.values()) + 1e-5)

    @classmethod
    def _ngram(cls, seq, n):
        for i in range(len(seq) - n + 1):
            yield seq[i : i + n]

    @classmethod
    def compute(cls, text, ngram=1):
        tokens = normalize_answer(text).split()
        return InterDistinctMetric(Counter(cls._ngram(tokens, ngram)))
```

OK, thanks! I'll test this.

Resolving patch:
diff --git a/parlai/core/metrics.py b/parlai/core/metrics.py
--- a/parlai/core/metrics.py
+++ b/parlai/core/metrics.py
@@ -12,7 +12,7 @@
 from collections import Counter
 import functools
 import datetime
-from typing import Union, List, Optional, Tuple, Set, Any, Dict
+from typing import Union, List, Optional, Tuple, Set, Any, Dict, Counter as TCounter
 
 import torch
 
@@ -23,7 +23,13 @@
 DEFAULT_METRICS = {'bleu-4', 'accuracy', 'f1'}
 ROUGE_METRICS = {'rouge-1', 'rouge-2', 'rouge-L'}
 BLEU_METRICS = {'bleu-1', 'bleu-2', 'bleu-3', 'bleu-4'}
-ALL_METRICS = DEFAULT_METRICS | ROUGE_METRICS | BLEU_METRICS
+DISTINCT_METRICS = {
+    'interdistinct-1',
+    'interdistinct-2',
+    'intradistinct-1',
+    'intradistinct-2',
+}
+ALL_METRICS = DEFAULT_METRICS | ROUGE_METRICS | BLEU_METRICS | DISTINCT_METRICS
 
 
 try:
@@ -523,6 +529,60 @@ def compute_many(
         )
 
 
+class IntraDistinctMetric(AverageMetric):
+    """
+    Compute intra-distinct (per-utterance).
+    """
+
+    @classmethod
+    def _ngram(cls, seq, n: int):
+        for i in range(len(seq) - n + 1):
+            yield tuple(seq[i : i + n])
+
+    @classmethod
+    def compute(cls, text: str, ngram: int = 1):
+        """
+        :param text:
+            The text to compute metric over
+        :param ngram:
+            n-gram length
+        """
+        tokens = normalize_answer(text).split()
+        counts = Counter(cls._ngram(tokens, ngram))
+        # computed per-example, macro averaged across examples
+        intra = max(len(counts), 1e-12) / max(sum(counts.values()), 1e-5)
+        return IntraDistinctMetric(intra, 1.0)
+
+
+class InterDistinctMetric(Metric):
+    """
+    Compute inter-distinct metric over corpus-level.
+    """
+
+    def __init__(self, counts: TCounter[Tuple]):
+        """
+        :param counts:
+            collections.Counter of ngram -> frequency
+        """
+        self._counts = counts
+
+    def __add__(self, other):
+        return InterDistinctMetric(self._counts + other._counts)
+
+    def value(self):
+        return max(len(self._counts), 1e-12) / max(sum(self._counts.values()), 1e-5)
+
+    @classmethod
+    def _ngram(cls, seq, n):
+        for i in range(len(seq) - n + 1):
+            yield tuple(seq[i : i + n])
+
+    @classmethod
+    def compute(cls, text, ngram=1):
+        tokens = normalize_answer(text).split()
+        return InterDistinctMetric(Counter(cls._ngram(tokens, ngram)))
+
+
 def normalize_answer(s):
     """
     Lower text and remove punctuation, articles and extra whitespace.
@@ -679,6 +739,8 @@ def _infer_metrics(cli_arg: str) -> Set[str]:
                 col |= ROUGE_METRICS
             elif n == 'bleu':
                 col |= BLEU_METRICS
+            elif n == 'distinct':
+                col |= DISTINCT_METRICS
             elif n == 'all':
                 col |= ALL_METRICS
             else:
@@ -732,6 +794,16 @@ def evaluate_response(self, observation: Message, labels: List[str]) -> None:
                     self.add('rouge_2', r2)
                 if 'rouge-L' in self._metrics_list and rL:
                     self.add('rouge_L', rL)
+            # compute distinct-k
+            for k in [1, 2]:
+                if f'interdistinct-{k}' in self._metrics_list:
+                    self.add(
+                        f'interdistinct-{k}', InterDistinctMetric.compute(prediction, k)
+                    )
+                if f'intradistinct-{k}' in self._metrics_list:
+                    self.add(
+                        f'intradistinct-{k}', IntraDistinctMetric.compute(prediction, k)
+                    )
 
         # Ranking metrics.
         self._update_ranking_metrics(observation, labels)


Test patch:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -18,6 +18,8 @@
     TimerMetric,
     aggregate_unnamed_reports,
     aggregate_named_reports,
+    InterDistinctMetric,
+    IntraDistinctMetric,
 )
 from parlai.core.torch_classifier_agent import ConfusionMatrixMetric, WeightedF1Metric
 
@@ -150,7 +152,7 @@ def test_multithreaded(self):
         m2.add('key', SumMetric(1))
         m3.add('key', SumMetric(2))
         m.add('key', SumMetric(3))
-        m.report()['key'] == 6
+        assert m.report()['key'] == 6
 
     def test_verymultithreaded(self):
         # legacy test, but useful all the same, for ensuring
@@ -385,5 +387,30 @@ def test_classifier_metrics(self):
         assert agg['weighted_f1'] == (0.5 + (2 / 3) * 0.5 + (4 / 7) * 0.5) / 2
 
 
+class TestDistinct(unittest.TestCase):
+    def test_inter_distinct(self):
+        # 3 n-grams, all appearing once
+        m = InterDistinctMetric.compute("this is some test", 2)
+        self.assertAlmostEqual(m, 1.0)
+        # 3-grams, each appearing twice
+        self.assertAlmostEqual(m + m, 0.5)
+
+    def test_inter_distinct_unigram(self):
+        m1 = InterDistinctMetric.compute("this test", 1)
+        self.assertAlmostEqual(m1, 1.0, delta=0.001)
+        m2 = InterDistinctMetric.compute("another test", 1)
+        self.assertAlmostEqual(m2, 1.0, delta=0.001)
+        # we now have 4 tokens, 3 words
+        self.assertAlmostEqual(m1 + m2, 3 / 4)
+
+    def test_intra_distinct(self):
+        # 4/5 are unique
+        m1 = IntraDistinctMetric.compute("this is some test test", 1)
+        self.assertAlmostEqual(m1, 4 / 5)
+        m2 = IntraDistinctMetric.compute("this test test test test", 1)
+        self.assertAlmostEqual(m2, 2 / 5)
+        self.assertAlmostEqual(m1 + m2, 3 / 5)
+
+
 if __name__ == '__main__':
     unittest.main()