Repository: obss/jury (language: python)

Issue:
Add CHRF
**Brief metric info**
chrF++ is a tool for automatic evaluation of machine translation output based on character n-gram precision and recall enhanced with word n-grams. The tool calculates the F-score averaged on all character and word n-grams, where the default character n-gram order is 6 and word n-gram order is 2. The arithmetic mean is used for n-gram averaging.

**Meta information**
Please fill below if applicable:
- Repo/Custom implementation: https://github.com/m-popovic/chrF
- Python implementation available: [x]
- Paper: https://aclanthology.org/W15-3049
- Huggingface: https://github.com/huggingface/datasets/tree/master/metrics/chrf



Discussion / hints:

Fix patch:
diff --git a/jury/metrics/__init__.py b/jury/metrics/__init__.py
index 853ff97..17b2c58 100644
--- a/jury/metrics/__init__.py
+++ b/jury/metrics/__init__.py
@@ -18,6 +18,7 @@
 from jury.metrics.bleu import Bleu
 from jury.metrics.bleurt import Bleurt
 from jury.metrics.cer import CER
+from jury.metrics.chrf import CHRF
 from jury.metrics.f1 import F1
 from jury.metrics.meteor import Meteor
 from jury.metrics.precision import Precision
diff --git a/jury/metrics/chrf/__init__.py b/jury/metrics/chrf/__init__.py
new file mode 100644
index 0000000..a07d2fa
--- /dev/null
+++ b/jury/metrics/chrf/__init__.py
@@ -0,0 +1,1 @@
+from jury.metrics.chrf.chrf import CHRF
diff --git a/jury/metrics/chrf/chrf.py b/jury/metrics/chrf/chrf.py
new file mode 100644
index 0000000..388e7b5
--- /dev/null
+++ b/jury/metrics/chrf/chrf.py
@@ -0,0 +1,8 @@
+from jury.metrics._core import MetricAlias
+from jury.metrics.chrf.chrf_for_language_generation import CHRFForLanguageGeneration
+
+__main_class__ = "CHRF"
+
+
+class CHRF(MetricAlias):
+    _SUBCLASS = CHRFForLanguageGeneration
diff --git a/jury/metrics/chrf/chrf_for_language_generation.py b/jury/metrics/chrf/chrf_for_language_generation.py
new file mode 100644
index 0000000..9ad976c
--- /dev/null
+++ b/jury/metrics/chrf/chrf_for_language_generation.py
@@ -0,0 +1,259 @@
+# coding=utf-8
+# Copyright 2020 Open Business Software Solutions, The HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" ChrF(++) (Character n-gram F-score) metric. The part of this file is adapted from HuggingFace's
+datasets package implementation of CHRF metric. See
+https://github.com/huggingface/datasets/blob/master/metrics/chrf/chrf.py
+"""
+from typing import Callable, Dict, List, Tuple, Union
+
+import datasets
+from packaging import version
+
+from jury.collator import Collator
+from jury.metrics import LanguageGenerationInstance, MetricForLanguageGeneration
+from jury.metrics._core.utils import PackagePlaceholder, requirement_message
+
+# `import sacrebleu as scb` placeholder
+scb = PackagePlaceholder(version="2.0.0")
+
+
+_CITATION = """\
+@inproceedings{popovic-2015-chrf,
+    title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation",
+    author = "Popovi{\'c}, Maja",
+    booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation",
+    month = sep,
+    year = "2015",
+    address = "Lisbon, Portugal",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/W15-3049",
+    doi = "10.18653/v1/W15-3049",
+    pages = "392--395",
+}
+@inproceedings{popovic-2017-chrf,
+    title = "chr{F}++: words helping character n-grams",
+    author = "Popovi{\'c}, Maja",
+    booktitle = "Proceedings of the Second Conference on Machine Translation",
+    month = sep,
+    year = "2017",
+    address = "Copenhagen, Denmark",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/W17-4770",
+    doi = "10.18653/v1/W17-4770",
+    pages = "612--618",
+}
+@inproceedings{post-2018-call,
+    title = "A Call for Clarity in Reporting {BLEU} Scores",
+    author = "Post, Matt",
+    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
+    month = oct,
+    year = "2018",
+    address = "Belgium, Brussels",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/W18-6319",
+    pages = "186--191",
+}
+"""
+
+_DESCRIPTION = """\
+ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches,
+and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation
+that is already present in sacrebleu.
+The implementation here is slightly different from sacrebleu in terms of the required input format. The length of
+the references and hypotheses lists need to be the same, so you may need to transpose your references compared to
+sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534
+See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information.
+"""
+
+_KWARGS_DESCRIPTION = """
+Produces ChrF(++) scores for hypotheses given reference translations.
+Args:
+    predictions: The system stream (a sequence of segments).
+    references: A list of one or more reference streams (each a sequence of segments).
+    char_order: Character n-gram order.
+    word_order: Word n-gram order. If equals to 2, the metric is referred to as chrF++.
+    beta: Determine the importance of recall w.r.t precision.
+    lowercase: Enable case-insensitivity.
+    whitespace: If `True`, include whitespaces when extracting character n-grams.
+    eps_smoothing: If `True`, applies epsilon smoothing similar
+    to reference chrF++.py, NLTK and Moses implementations. Otherwise,
+    it takes into account effective match order similar to sacreBLEU < 2.0.0.
+Returns:
+    'score': The chrF (chrF++) score,
+    'char_order': The character n-gram order,
+    'word_order': The word n-gram order. If equals to 2, the metric is referred to as chrF++,
+    'beta': Determine the importance of recall w.r.t precision
+Examples:
+    >>> prediction = [["The relationship between Obama and Netanyahu is not exactly friendly."]]
+    >>> reference = [["The ties between Obama and Netanyahu are not particularly friendly."]]
+    >>> chrf = jury.load_metric("chrf")
+    >>> results = chrf.compute(predictions=prediction, references=reference)
+    >>> print(results)
+    {'score': 61.576379378113785, 'char_order': 6, 'word_order': 0, 'beta': 2}
+"""
+
+
+@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class CHRFForLanguageGeneration(MetricForLanguageGeneration):
+    def _info(self):
+        if version.parse(scb.__version__) < version.parse("1.4.12"):
+            raise ImportWarning(
+                "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of "
+                "`sacrebleu` doesn't match this condition.\nYou can install it with `pip install sacrebleu>=1.4.12`."
+            )
+        return datasets.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            homepage="https://github.com/mjpost/sacreBLEU#chrf--chrf",
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=self._default_features,
+            codebase_urls=["https://github.com/mjpost/sacreBLEU#chrf--chrf"],
+            reference_urls=[
+                "https://github.com/m-popovic/chrF",
+            ],
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        global scb
+        global CHRFScorer
+        try:
+            from sacrebleu import CHRF as CHRFScorer
+        except ModuleNotFoundError:
+            raise ModuleNotFoundError(requirement_message(path="chrf", package_name="sacrebleu"))
+        else:
+            super(CHRFForLanguageGeneration, self)._download_and_prepare(dl_manager)
+
+    def _validate_references(self, references: Collator) -> None:
+        references_per_prediction = len(references[0])
+        if any(len(refs) != references_per_prediction for refs in references):
+            raise ValueError("Sacrebleu requires the same number of references for each prediction")
+
+    def _compute_chrf_score(
+        self, predictions: Union[str, List[str]], references: Union[str, List[str]], **kwargs
+    ) -> Tuple[float, int, int, int]:
+        if kwargs.get("char_order") is None:
+            kwargs["char_order"] = CHRFScorer.CHAR_ORDER
+        if kwargs.get("word_order") is None:
+            kwargs["word_order"] = CHRFScorer.WORD_ORDER
+        if kwargs.get("beta") is None:
+            kwargs["beta"] = CHRFScorer.BETA
+        sb_chrf = CHRFScorer(**kwargs)
+        output = sb_chrf.corpus_score(predictions, references)
+
+        return (
+            output.score / 100,
+            output.char_order,
+            output.word_order,
+            output.beta,
+        )
+
+    def _compute_single_pred_single_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        char_order: int = None,
+        word_order: int = None,
+        beta: int = None,
+        lowercase: bool = False,
+        whitespace: bool = False,
+        eps_smoothing: bool = False,
+    ):
+        score, c_ord, w_ord, beta = self._compute_chrf_score(
+            predictions,
+            references,
+            char_order=char_order,
+            word_order=word_order,
+            beta=beta,
+            lowercase=lowercase,
+            whitespace=whitespace,
+            eps_smoothing=eps_smoothing,
+        )
+        return {"score": score, "char_order": c_ord, "word_order": w_ord, "beta": beta}
+
+    def _compute_single_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        char_order: int = None,
+        word_order: int = None,
+        beta: int = None,
+        lowercase: bool = False,
+        whitespace: bool = False,
+        eps_smoothing: bool = False,
+    ):
+        # SacreBleu inherently supports multiple references.
+        return self._compute_single_pred_single_ref(
+            predictions=predictions,
+            references=references,
+            reduce_fn=reduce_fn,
+            char_order=char_order,
+            word_order=word_order,
+            beta=beta,
+            lowercase=lowercase,
+            whitespace=whitespace,
+            eps_smoothing=eps_smoothing,
+        )
+
+    def _compute_multi_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        char_order: int = None,
+        word_order: int = None,
+        beta: int = None,
+        lowercase: bool = False,
+        whitespace: bool = False,
+        eps_smoothing: bool = False,
+    ):
+        scores = []
+        for preds, refs in zip(predictions, references):
+            pred_scores = []
+            for pred in preds:
+                score, _, _, _ = self._compute_chrf_score(
+                    pred,
+                    refs,
+                    char_order=char_order,
+                    word_order=word_order,
+                    beta=beta,
+                    lowercase=lowercase,
+                    whitespace=whitespace,
+                    eps_smoothing=eps_smoothing,
+                )
+                pred_scores.append(score)
+            scores.append(float(reduce_fn(pred_scores)))
+
+        return {
+            "score": sum(scores) / len(scores),
+            "char_order": char_order or CHRFScorer.CHAR_ORDER,
+            "word_order": word_order or CHRFScorer.WORD_ORDER,
+            "beta": beta or CHRFScorer.BETA,
+        }
+
+    def evaluate(
+        self, predictions: Collator, references: Collator, reduce_fn: Callable = None, **kwargs
+    ) -> Dict[str, float]:
+        if predictions.can_collapse() and references.can_collapse():
+            predictions = predictions.collapse()
+            eval_fn = self._compute_single_pred_single_ref
+        elif predictions.can_collapse() and not references.can_collapse():
+            predictions = predictions.collapse()
+            eval_fn = self._compute_single_pred_multi_ref
+        else:
+            eval_fn = self._compute_multi_pred_multi_ref
+        self._validate_references(references)
+        return eval_fn(predictions=predictions, references=references, reduce_fn=reduce_fn, **kwargs)


Test patch:
diff --git a/tests/jury/metrics/test_chrf.py b/tests/jury/metrics/test_chrf.py
new file mode 100644
index 0000000..2dfecdb
--- /dev/null
+++ b/tests/jury/metrics/test_chrf.py
@@ -0,0 +1,47 @@
+import pytest
+
+from jury import Jury
+from jury.metrics import AutoMetric
+from tests.jury.conftest import get_expected_output
+from tests.utils import assert_almost_equal_dict
+
+
+@pytest.fixture(scope="module")
+def jury_chrf():
+    metric = AutoMetric.load("chrf")
+    return Jury(metrics=metric)
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_basic():
+    return output_basic.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_ref():
+    return output_multiple_ref.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_pred_multiple_ref():
+    return output_multiple_pred_multiple_ref.output
+
+
+def test_basic(predictions, references, jury_chrf, output_basic):
+    scores = jury_chrf(predictions=predictions, references=references)
+    assert_almost_equal_dict(actual=scores, desired=output_basic)
+
+
+def test_multiple_ref(predictions, multiple_references, jury_chrf, output_multiple_ref):
+    scores = jury_chrf(predictions=predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_ref)
+
+
+def test_multiple_pred_multiple_ref(
+    multiple_predictions, multiple_references, jury_chrf, output_multiple_pred_multiple_ref
+):
+    scores = jury_chrf(predictions=multiple_predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_pred_multiple_ref)
diff --git a/tests/test_data/expected_outputs/metrics/test_chrf.json b/tests/test_data/expected_outputs/metrics/test_chrf.json
new file mode 100644
index 0000000..3699403
--- /dev/null
+++ b/tests/test_data/expected_outputs/metrics/test_chrf.json
@@ -0,0 +1,32 @@
+{
+  "basic": {
+    "total_items": 2,
+    "empty_items": 0,
+    "chrf": {
+      "score": 0.4272182337289785,
+      "char_order": 6,
+      "word_order": 0,
+      "beta": 2
+    }
+  },
+  "multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "chrf": {
+      "score": 0.340817184979931,
+      "char_order": 6,
+      "word_order": 0,
+      "beta": 2
+    }
+  },
+  "multiple_pred_multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "chrf": {
+      "score": 0.4429840574418887,
+      "char_order": 6,
+      "word_order": 0,
+      "beta": 2
+    }
+  }
+}
\ No newline at end of file