GitHub issue #3187: Add ChrF(++) (as implemented in sacrebleu)
State: closed | Created: 2021-10-31T08:53:58Z

Similar to my [PR for TER](https://github.com/huggingface/datasets/pull/3153), it feels only right to also include ChrF and friends. These are present in Sacrebleu and are therefore very similar to implement as TER and sacrebleu. I tested the implementation with sacrebleu's tests to verify. You can try this below for yourself

```python
import datasets


EPSILON = 1e-4
chrf = datasets.load_metric(r"path\to\datasets\metrics\chrf")

test_cases = [
    (["abcdefg"], ["hijklmnop"], 0.0),
    (["a"], ["b"], 0.0),
    ([""], ["b"], 0.0),
    ([""], ["ref"], 0.0),
    ([""], ["reference"], 0.0),
    (["aa"], ["ab"], 8.3333),
    (["a", "b"], ["a", "c"], 8.3333),
    (["a"], ["a"], 16.6667),
    (["a b c"], ["a b c"], 50.0),
    (["a b c"], ["abc"], 50.0),
    ([" risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists ."],
     ["risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists ."], 63.361730),
    ([" Die    Beziehung zwischen  Obama und Netanjahu ist nicht gerade  freundlich. "],
     ["Das Verhältnis zwischen Obama und Netanyahu ist nicht gerade freundschaftlich."], 64.1302698),
    (["Niemand hat die Absicht, eine Mauer zu errichten"], ["Niemand hat die Absicht, eine Mauer zu errichten"], 100.0),
]

for hyp, ref, score in test_cases:
    # Note the reference transformation which is different from scarebleu's input format
    results = chrf.compute(predictions=hyp, references=[[r] for r in ref],
                           char_order=6, word_order=0, beta=3, eps_smoothing=True)
    if abs(score - results["score"]) > EPSILON:
        print(f"expected {score}, got {results['score']} for {hyp} - {ref}")

test_cases_effective_order = [
    (["a"], ["a"], 100.0),
    ([""], ["reference"], 0.0),
    (["a b c"], ["a b c"], 100.0),
    (["a b c"], ["abc"], 100.0),
    ([""], ["c"], 0.0),
    (["a", "b"], ["a", "c"], 50.0),
    (["aa"], ["ab"], 25.0),
]

for hyp, ref, score in test_cases_effective_order:
    # Note the reference transformation which is different from scarebleu's input format
    results = chrf.compute(predictions=hyp, references=[[r] for r in ref],
                           char_order=6, word_order=0, beta=3, eps_smoothing=False)
    if abs(score - results["score"]) > EPSILON:
        print(f"expected {score}, got {results['score']} for {hyp} - {ref}")

test_cases_keep_whitespace = [
    (
        ["Die Beziehung zwischen Obama und Netanjahu ist nicht gerade freundlich."],
        ["Das Verhältnis zwischen Obama und Netanyahu ist nicht gerade freundschaftlich."],
        67.3481606,
    ),
    (
        ["risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists ."],
        ["risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists ."],
        65.2414427,
    ),
]

for hyp, ref, score in test_cases_keep_whitespace:
    # Note the reference transformation which is different from scarebleu's input format
    results = chrf.compute(predictions=hyp, references=[[r] for r in ref],
                           char_order=6, word_order=0, beta=3,
                           whitespace=True)
    if abs(score - results["score"]) > EPSILON:
        print(f"expected {score}, got {results['score']} for {hyp} - {ref}")


predictions = ["The relationship between Obama and Netanyahu is not exactly friendly."]
references = [["The ties between Obama and Netanyahu are not particularly friendly."]]
print(chrf.compute(predictions=predictions, references=references))
```