Repository: obss/jury (language: python)

Issue:
New Metric: CER
CER is currently available at HF datasets and implicitly available also at jury. Yet, to fully utilize jury, it needs to be added.

HF Implementation: https://github.com/huggingface/datasets/blob/master/metrics/cer/cer.py


Discussion / hints:

Fix patch:
diff --git a/jury/metrics/__init__.py b/jury/metrics/__init__.py
index f7a8402..54b01fa 100644
--- a/jury/metrics/__init__.py
+++ b/jury/metrics/__init__.py
@@ -16,6 +16,7 @@
 from jury.metrics.bertscore import Bertscore
 from jury.metrics.bleu import Bleu
 from jury.metrics.bleurt import Bleurt
+from jury.metrics.cer import CER
 from jury.metrics.f1 import F1
 from jury.metrics.meteor import Meteor
 from jury.metrics.precision import Precision
diff --git a/jury/metrics/cer/__init__.py b/jury/metrics/cer/__init__.py
new file mode 100644
index 0000000..f03a303
--- /dev/null
+++ b/jury/metrics/cer/__init__.py
@@ -0,0 +1,1 @@
+from jury.metrics.cer.cer import CER
diff --git a/jury/metrics/cer/cer.py b/jury/metrics/cer/cer.py
new file mode 100644
index 0000000..27e9935
--- /dev/null
+++ b/jury/metrics/cer/cer.py
@@ -0,0 +1,8 @@
+from jury.metrics._core import MetricAlias
+from jury.metrics.cer.cer_for_language_generation import CERForLanguageGeneration
+
+__main_class__ = "CER"
+
+
+class CER(MetricAlias):
+    _SUBCLASS = CERForLanguageGeneration
diff --git a/jury/metrics/cer/cer_for_language_generation.py b/jury/metrics/cer/cer_for_language_generation.py
new file mode 100644
index 0000000..f250fb7
--- /dev/null
+++ b/jury/metrics/cer/cer_for_language_generation.py
@@ -0,0 +1,249 @@
+# coding=utf-8
+# Copyright 2020 Open Business Software Solutions, The HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" CER metric. The part of this file is adapted from HuggingFace's
+datasets package implementation of CER metric. See
+https://github.com/huggingface/datasets/blob/master/metrics/cer/cer.py
+"""
+import warnings
+from typing import Callable, List, Tuple, Union
+
+import datasets
+
+from jury.metrics import LanguageGenerationInstance, MetricForLanguageGeneration
+from jury.metrics._core.utils import PackagePlaceholder, requirement_message
+
+# `import jiwer` placeholder
+jiwer = PackagePlaceholder(version="2.3.0")
+
+
+_CITATION = """\
+@inproceedings{inproceedings,
+    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
+    year = {2004},
+    month = {01},
+    pages = {},
+    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
+}
+"""
+
+_DESCRIPTION = """\
+Character error rate (CER) is a common metric of the performance of an automatic speech recognition system.
+CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.
+Character error rate can be computed as:
+CER = (S + D + I) / N = (S + D + I) / (S + D + C)
+where
+S is the number of substitutions,
+D is the number of deletions,
+I is the number of insertions,
+C is the number of correct characters,
+N is the number of characters in the reference (N=S+D+C).
+CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the
+performance of the ASR system with a CER of 0 being a perfect score.
+"""
+
+_KWARGS_DESCRIPTION = """
+Computes CER score of transcribed segments against references.
+Args:
+    references: list of references for each speech input.
+    predictions: list of transcribtions to score.
+    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.
+Returns:
+    (float): the character error rate
+Examples:
+    >>> predictions = ["this is the prediction", "there is an other sample"]
+    >>> references = ["this is the reference", "there is another one"]
+    >>> cer = jury.load_metric("cer")
+    >>> cer_score = cer.compute(predictions=predictions, references=references)
+    >>> print(cer_score)
+    0.34146341463414637
+"""
+
+
+@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class CERForLanguageGeneration(MetricForLanguageGeneration):
+    def _info(self):
+        return datasets.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=self._default_features,
+            codebase_urls=["https://github.com/jitsi/jiwer/"],
+            reference_urls=[
+                "https://en.wikipedia.org/wiki/Word_error_rate",
+                "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates",
+            ],
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        global jiwer
+        global tr
+
+        try:
+            import jiwer
+            import jiwer.transforms as tr
+        except ModuleNotFoundError:
+            raise ModuleNotFoundError(requirement_message(path="CER", package_name="jiwer"))
+        else:
+            super(CERForLanguageGeneration, self)._download_and_prepare(dl_manager)
+
+    @staticmethod
+    def _get_cer_transform():
+        return tr.Compose(
+            [
+                tr.RemoveMultipleSpaces(),
+                tr.Strip(),
+                tr.ReduceToSingleSentence(""),
+                tr.ReduceToListOfListOfChars(),
+            ]
+        )
+
+    def _compute_cer_score(
+        self, predictions: Union[str, List[str]], references: Union[str, List[str]]
+    ) -> Tuple[float, int, int, int, int]:
+        cer_transform = self._get_cer_transform()
+        measures = jiwer.compute_measures(
+            references,
+            predictions,
+            truth_transform=cer_transform,
+            hypothesis_transform=cer_transform,
+        )
+        return (
+            measures["wer"],
+            int(measures["substitutions"]),
+            int(measures["deletions"]),
+            int(measures["insertions"]),
+            int(measures["hits"]),
+        )
+
+    def _compute_single_pred_single_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        concatenate_texts: bool = False,
+    ):
+        if concatenate_texts:
+            score, total_substitutions, total_deletions, total_insertions, total_hits = self._compute_cer_score(
+                predictions, references
+            )
+        else:
+            incorrect = 0
+            total = 0
+            total_substitutions = 0
+            total_deletions = 0
+            total_insertions = 0
+            total_hits = 0
+            for prediction, reference in zip(predictions, references):
+                _, substitutions, deletions, insertions, hits = self._compute_cer_score(prediction, reference)
+                total_substitutions += substitutions
+                total_deletions += deletions
+                total_insertions += insertions
+                total_hits += hits
+                incorrect += substitutions + deletions + insertions
+                total += substitutions + deletions + hits
+            score = incorrect / total
+
+        return {
+            "score": score,
+            "overall": {
+                "substitutions": total_substitutions,
+                "deletions": total_deletions,
+                "insertions": total_insertions,
+                "hits": total_hits,
+            },
+        }
+
+    def _compute_single_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        **kwargs
+    ):
+        if "concatenate_texts" in kwargs:
+            warnings.warn("Option 'concatenate_texts' is only available in single-pred & single-ref setting.")
+
+        total_substitutions = 0
+        total_deletions = 0
+        total_insertions = 0
+        total_hits = 0
+        total_refs = 0
+        scores = []
+        for pred, refs in zip(predictions, references):
+            pred_scores = []
+            for ref in refs:
+                score, substitutions, deletions, insertions, hits = self._compute_cer_score(
+                    predictions=pred, references=ref
+                )
+                pred_scores.append(score)
+                total_substitutions += substitutions
+                total_deletions += deletions
+                total_insertions += insertions
+                total_hits += hits
+                total_refs += 1
+            reduced_score = float(reduce_fn(pred_scores))
+            scores.append(reduced_score)
+
+        return {
+            "score": float(reduce_fn(scores)),
+            "overall": {
+                "substitutions": total_substitutions / total_refs,
+                "deletions": total_deletions / total_refs,
+                "insertions": total_insertions / total_refs,
+                "hits": total_hits / total_refs,
+            },
+        }
+
+    def _compute_multi_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        **kwargs
+    ):
+        if "concatenate_texts" in kwargs:
+            warnings.warn("Option 'concatenate_texts' is only available in single-pred & single-ref setting.")
+
+        total_substitutions = 0
+        total_deletions = 0
+        total_insertions = 0
+        total_hits = 0
+        total_refs = 0
+        scores = []
+        for preds, refs in zip(predictions, references):
+            pred_scores = []
+            for pred in preds:
+                for ref in refs:
+                    score, substitutions, deletions, insertions, hits = self._compute_cer_score(
+                        predictions=pred, references=ref
+                    )
+                    pred_scores.append(score)
+                    total_substitutions += substitutions
+                    total_deletions += deletions
+                    total_insertions += insertions
+                    total_hits += hits
+                    total_refs += 1
+            reduced_score = float(reduce_fn(pred_scores))
+            scores.append(reduced_score)
+
+        return {
+            "score": float(reduce_fn(scores)),
+            "overall": {
+                "substitutions": total_substitutions / total_refs,
+                "deletions": total_deletions / total_refs,
+                "insertions": total_insertions / total_refs,
+                "hits": total_hits / total_refs,
+            },
+        }
diff --git a/setup.py b/setup.py
index 8a969d5..f083c20 100644
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,6 @@ def add_pywin(reqs: List[str]) -> None:
     "deepdiff==5.5.0",
     "flake8==3.9.2",
     "isort==5.9.2",
-    "jiwer>=2.2.0",
     "pytest>=6.2.4",
     "pytest-cov>=2.12.1",
     "pytest-timeout>=1.4.2",
@@ -50,9 +49,10 @@ def add_pywin(reqs: List[str]) -> None:
 _METRIC_REQUIREMENTS = [
     "sacrebleu>=2.0.0",
     "bert_score==0.3.11",
+    "bleurt @ git+https://github.com/google-research/bleurt.git",
+    "jiwer>=2.3.0",
     "seqeval==1.2.2",
     "sentencepiece==0.1.96",
-    "bleurt @ git+https://github.com/google-research/bleurt.git",
 ]
 _METRIC_REQUIREMENTS.extend(_PRISM_REQUIREMENTS)
 add_pywin(_METRIC_REQUIREMENTS)


Test patch:
diff --git a/tests/jury/metrics/test_cer.py b/tests/jury/metrics/test_cer.py
new file mode 100644
index 0000000..3e52660
--- /dev/null
+++ b/tests/jury/metrics/test_cer.py
@@ -0,0 +1,62 @@
+import pytest
+
+from jury import Jury
+from jury.metrics import AutoMetric
+from tests.jury.conftest import get_expected_output
+from tests.utils import assert_almost_equal_dict
+
+
+@pytest.fixture(scope="module")
+def jury():
+    metric = AutoMetric.load("cer")
+    return Jury(metrics=metric)
+
+
+@pytest.fixture(scope="module")
+def jury_concat():
+    metric = AutoMetric.load("cer", compute_kwargs={"concatenate_texts": True})
+    return Jury(metrics=metric)
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_basic():
+    return output_basic.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_basic_concat():
+    return output_basic_concat.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_ref():
+    return output_multiple_ref.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_pred_multiple_ref():
+    return output_multiple_pred_multiple_ref.output
+
+
+def test_basic(predictions, references, jury, output_basic):
+    scores = jury(predictions=predictions, references=references)
+    assert_almost_equal_dict(actual=scores, desired=output_basic)
+
+
+def test_basic_concat(predictions, references, jury_concat, output_basic_concat):
+    scores = jury_concat(predictions=predictions, references=references)
+    assert_almost_equal_dict(actual=scores, desired=output_basic_concat)
+
+
+def test_multiple_ref(predictions, multiple_references, jury, output_multiple_ref):
+    scores = jury(predictions=predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_ref)
+
+
+def test_multiple_pred_multiple_ref(multiple_predictions, multiple_references, jury, output_multiple_pred_multiple_ref):
+    scores = jury(predictions=multiple_predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_pred_multiple_ref)
diff --git a/tests/test_data/expected_outputs/metrics/test_cer.json b/tests/test_data/expected_outputs/metrics/test_cer.json
new file mode 100644
index 0000000..ec3eed6
--- /dev/null
+++ b/tests/test_data/expected_outputs/metrics/test_cer.json
@@ -0,0 +1,54 @@
+{
+  "basic": {
+    "total_items": 2,
+    "empty_items": 0,
+    "cer": {
+      "score": 0.3333333333333333,
+      "overall": {
+        "substitutions": 10,
+        "deletions": 7,
+        "insertions": 1,
+        "hits": 37
+      }
+    }
+  },
+  "basic_concat": {
+    "total_items": 2,
+    "empty_items": 0,
+    "cer": {
+      "score": 0.3333333333333333,
+      "overall": {
+        "substitutions": 10,
+        "deletions": 7,
+        "insertions": 1,
+        "hits": 37
+      }
+    }
+  },
+  "multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "cer": {
+      "score": 0.7272727272727273,
+      "overall": {
+        "substitutions": 6.25,
+        "deletions": 5.5,
+        "insertions": 1.5,
+        "hits": 16.25
+      }
+    }
+  },
+  "multiple_pred_multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "cer": {
+      "score": 0.8484848484848485,
+      "overall": {
+        "substitutions": 6.625,
+        "deletions": 3.875,
+        "insertions": 3.625,
+        "hits": 17.5
+      }
+    }
+  }
+}