Repository: obss/jury (language: python)

Issue:
Add WER
**Brief metric info**
Word Error Rate metric a derivation of edit distance.

**Meta information**
Please fill below if applicable:
- Repo/Custom implementation: -
- Python implementation available: [x]
- Paper: -
- Huggingface: https://github.com/huggingface/datasets/tree/master/metrics/wer



Discussion / hints:

Fix patch:
diff --git a/jury/metrics/__init__.py b/jury/metrics/__init__.py
index a5a5858..853ff97 100644
--- a/jury/metrics/__init__.py
+++ b/jury/metrics/__init__.py
@@ -17,6 +17,7 @@
 from jury.metrics.bertscore import Bertscore
 from jury.metrics.bleu import Bleu
 from jury.metrics.bleurt import Bleurt
+from jury.metrics.cer import CER
 from jury.metrics.f1 import F1
 from jury.metrics.meteor import Meteor
 from jury.metrics.precision import Precision
@@ -25,3 +26,4 @@
 from jury.metrics.rouge import Rouge
 from jury.metrics.sacrebleu import Sacrebleu
 from jury.metrics.squad import Squad
+from jury.metrics.wer import WER
diff --git a/jury/metrics/wer/__init__.py b/jury/metrics/wer/__init__.py
new file mode 100644
index 0000000..3a9f89d
--- /dev/null
+++ b/jury/metrics/wer/__init__.py
@@ -0,0 +1,1 @@
+from jury.metrics.wer.wer import WER
diff --git a/jury/metrics/wer/wer.py b/jury/metrics/wer/wer.py
new file mode 100644
index 0000000..f186cb1
--- /dev/null
+++ b/jury/metrics/wer/wer.py
@@ -0,0 +1,8 @@
+from jury.metrics._core import MetricAlias
+from jury.metrics.wer.wer_for_language_generation import WERForLanguageGeneration
+
+__main_class__ = "WER"
+
+
+class WER(MetricAlias):
+    _SUBCLASS = WERForLanguageGeneration
diff --git a/jury/metrics/wer/wer_for_language_generation.py b/jury/metrics/wer/wer_for_language_generation.py
new file mode 100644
index 0000000..c4360ed
--- /dev/null
+++ b/jury/metrics/wer/wer_for_language_generation.py
@@ -0,0 +1,239 @@
+# coding=utf-8
+# Copyright 2020 Open Business Software Solutions, The HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Word Error Ratio (WER) metric. The part of this file is adapted from HuggingFace's
+datasets package implementation of CER metric. See
+https://github.com/huggingface/datasets/blob/master/metrics/wer/wer.py
+"""
+import warnings
+from typing import Callable, List, Tuple, Union
+
+import datasets
+
+from jury.metrics import LanguageGenerationInstance, MetricForLanguageGeneration
+from jury.metrics._core.utils import PackagePlaceholder, requirement_message
+
+# `import jiwer` placeholder
+jiwer = PackagePlaceholder(version="2.3.0")
+
+
+_CITATION = """\
+@inproceedings{inproceedings,
+    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
+    year = {2004},
+    month = {01},
+    pages = {},
+    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
+}
+"""
+
+_DESCRIPTION = """\
+Word error rate (WER) is a common metric of the performance of an automatic speech recognition system.
+The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort.
+This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate.
+Word error rate can then be computed as:
+WER = (S + D + I) / N = (S + D + I) / (S + D + C)
+where
+S is the number of substitutions,
+D is the number of deletions,
+I is the number of insertions,
+C is the number of correct words,
+N is the number of words in the reference (N=S+D+C).
+This value indicates the average number of errors per reference word. The lower the value, the better the
+performance of the ASR system with a WER of 0 being a perfect score.
+"""
+
+_KWARGS_DESCRIPTION = """
+Compute WER score of transcribed segments against references.
+Args:
+    references: List of references for each speech input.
+    predictions: List of transcriptions to score.
+    concatenate_texts (bool, default=False): Whether to concatenate all input texts or compute WER iteratively.
+Returns:
+    (float): the word error rate
+Examples:
+    >>> predictions = ["this is the prediction", "there is an other sample"]
+    >>> references = ["this is the reference", "there is another one"]
+    >>> wer = jury.load_metric("wer")
+    >>> wer_score = wer.compute(predictions=predictions, references=references)
+    >>> print(wer_score)
+    {
+      "score": 0.5,
+      "overall": {
+        "substitutions": 3,
+        "deletions": 0,
+        "insertions": 1,
+        "hits": 5
+      }
+    }
+"""
+
+
+@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class WERForLanguageGeneration(MetricForLanguageGeneration):
+    def _info(self):
+        return datasets.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=self._default_features,
+            codebase_urls=["https://github.com/jitsi/jiwer/"],
+            reference_urls=[
+                "https://en.wikipedia.org/wiki/Word_error_rate",
+                "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates",
+            ],
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        global jiwer
+
+        try:
+            import jiwer
+        except ModuleNotFoundError:
+            raise ModuleNotFoundError(requirement_message(path="WER", package_name="jiwer"))
+        else:
+            super(WERForLanguageGeneration, self)._download_and_prepare(dl_manager)
+
+    def _compute_wer_score(
+        self, predictions: Union[str, List[str]], references: Union[str, List[str]]
+    ) -> Tuple[float, int, int, int, int]:
+        measures = jiwer.compute_measures(references, predictions)
+        return (
+            measures["wer"],
+            int(measures["substitutions"]),
+            int(measures["deletions"]),
+            int(measures["insertions"]),
+            int(measures["hits"]),
+        )
+
+    def _compute_single_pred_single_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        concatenate_texts: bool = False,
+    ):
+        if concatenate_texts:
+            score, total_substitutions, total_deletions, total_insertions, total_hits = self._compute_wer_score(
+                predictions, references
+            )
+        else:
+            incorrect = 0
+            total = 0
+            total_substitutions = 0
+            total_deletions = 0
+            total_insertions = 0
+            total_hits = 0
+            for prediction, reference in zip(predictions, references):
+                _, substitutions, deletions, insertions, hits = self._compute_wer_score(prediction, reference)
+                total_substitutions += substitutions
+                total_deletions += deletions
+                total_insertions += insertions
+                total_hits += hits
+                incorrect += substitutions + deletions + insertions
+                total += substitutions + deletions + hits
+            score = incorrect / total
+
+        return {
+            "score": score,
+            "overall": {
+                "substitutions": total_substitutions,
+                "deletions": total_deletions,
+                "insertions": total_insertions,
+                "hits": total_hits,
+            },
+        }
+
+    def _compute_single_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        **kwargs
+    ):
+        if "concatenate_texts" in kwargs:
+            warnings.warn("Option 'concatenate_texts' is only available in single-pred & single-ref setting.")
+
+        total_substitutions = 0
+        total_deletions = 0
+        total_insertions = 0
+        total_hits = 0
+        total_refs = 0
+        scores = []
+        for pred, refs in zip(predictions, references):
+            pred_scores = []
+            for ref in refs:
+                score, substitutions, deletions, insertions, hits = self._compute_wer_score(
+                    predictions=pred, references=ref
+                )
+                pred_scores.append(score)
+                total_substitutions += substitutions
+                total_deletions += deletions
+                total_insertions += insertions
+                total_hits += hits
+                total_refs += 1
+            reduced_score = float(reduce_fn(pred_scores))
+            scores.append(reduced_score)
+
+        return {
+            "score": float(reduce_fn(scores)),
+            "overall": {
+                "substitutions": total_substitutions / total_refs,
+                "deletions": total_deletions / total_refs,
+                "insertions": total_insertions / total_refs,
+                "hits": total_hits / total_refs,
+            },
+        }
+
+    def _compute_multi_pred_multi_ref(
+        self,
+        predictions: LanguageGenerationInstance,
+        references: LanguageGenerationInstance,
+        reduce_fn: Callable = None,
+        **kwargs
+    ):
+        if "concatenate_texts" in kwargs:
+            warnings.warn("Option 'concatenate_texts' is only available in single-pred & single-ref setting.")
+
+        total_substitutions = 0
+        total_deletions = 0
+        total_insertions = 0
+        total_hits = 0
+        total_refs = 0
+        scores = []
+        for preds, refs in zip(predictions, references):
+            pred_scores = []
+            for pred in preds:
+                for ref in refs:
+                    score, substitutions, deletions, insertions, hits = self._compute_wer_score(
+                        predictions=pred, references=ref
+                    )
+                    pred_scores.append(score)
+                    total_substitutions += substitutions
+                    total_deletions += deletions
+                    total_insertions += insertions
+                    total_hits += hits
+                    total_refs += 1
+            reduced_score = float(reduce_fn(pred_scores))
+            scores.append(reduced_score)
+
+        return {
+            "score": float(reduce_fn(scores)),
+            "overall": {
+                "substitutions": total_substitutions / total_refs,
+                "deletions": total_deletions / total_refs,
+                "insertions": total_insertions / total_refs,
+                "hits": total_hits / total_refs,
+            },
+        }
diff --git a/setup.py b/setup.py
index ef01e67..d77400d 100644
--- a/setup.py
+++ b/setup.py
@@ -42,6 +42,7 @@ def add_pywin(reqs: List[str]) -> None:
     "pytest>=7.0.1",
     "pytest-cov>=3.0.0",
     "pytest-timeout>=2.1.0",
+    "math_equivalence @ git+https://github.com/hendrycks/math.git",  # for datasets test metric
 ]
 
 _PRISM_REQUIREMENTS = ["fairseq==0.9.0", "validators"]


Test patch:
diff --git a/tests/jury/conftest.py b/tests/jury/conftest.py
index 6385432..fc92be4 100644
--- a/tests/jury/conftest.py
+++ b/tests/jury/conftest.py
@@ -64,7 +64,7 @@
     {"path": "squad"},
 ]
 
-_DATASETS_METRICS = "wer"
+_DATASETS_METRICS = "competition_math"
 
 _TEST_METRICS_SEQUENCE_CLASSIFICATION = [
     {"path": "accuracy", "task": "sequence-classification"},
diff --git a/tests/jury/metrics/test_wer.py b/tests/jury/metrics/test_wer.py
new file mode 100644
index 0000000..966d5c5
--- /dev/null
+++ b/tests/jury/metrics/test_wer.py
@@ -0,0 +1,64 @@
+import pytest
+
+from jury import Jury
+from jury.metrics import AutoMetric
+from tests.jury.conftest import get_expected_output
+from tests.utils import assert_almost_equal_dict
+
+
+@pytest.fixture(scope="module")
+def jury_wer():
+    metric = AutoMetric.load("wer")
+    return Jury(metrics=metric)
+
+
+@pytest.fixture(scope="module")
+def jury_wer_concat():
+    metric = AutoMetric.load("wer", compute_kwargs={"concatenate_texts": True})
+    return Jury(metrics=metric)
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_basic():
+    return output_basic.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_basic_concat():
+    return output_basic_concat.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_ref():
+    return output_multiple_ref.output
+
+
+@pytest.fixture
+@get_expected_output(prefix="metrics")
+def output_multiple_pred_multiple_ref():
+    return output_multiple_pred_multiple_ref.output
+
+
+def test_basic(predictions, references, jury_wer, output_basic):
+    scores = jury_wer(predictions=predictions, references=references)
+    assert_almost_equal_dict(actual=scores, desired=output_basic)
+
+
+def test_basic_concat(predictions, references, jury_wer_concat, output_basic_concat):
+    scores = jury_wer_concat(predictions=predictions, references=references)
+    assert_almost_equal_dict(actual=scores, desired=output_basic_concat)
+
+
+def test_multiple_ref(predictions, multiple_references, jury_wer, output_multiple_ref):
+    scores = jury_wer(predictions=predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_ref)
+
+
+def test_multiple_pred_multiple_ref(
+    multiple_predictions, multiple_references, jury_wer, output_multiple_pred_multiple_ref
+):
+    scores = jury_wer(predictions=multiple_predictions, references=multiple_references)
+    assert_almost_equal_dict(actual=scores, desired=output_multiple_pred_multiple_ref)
diff --git a/tests/test_data/expected_outputs/metrics/test_wer.json b/tests/test_data/expected_outputs/metrics/test_wer.json
new file mode 100644
index 0000000..7be8e5f
--- /dev/null
+++ b/tests/test_data/expected_outputs/metrics/test_wer.json
@@ -0,0 +1,54 @@
+{
+  "basic": {
+    "total_items": 2,
+    "empty_items": 0,
+    "wer": {
+      "score": 0.5833333333333334,
+      "overall": {
+        "substitutions": 6,
+        "deletions": 1,
+        "insertions": 0,
+        "hits": 5
+      }
+    }
+  },
+  "basic_concat": {
+    "total_items": 2,
+    "empty_items": 0,
+    "wer": {
+      "score": 0.5833333333333334,
+      "overall": {
+        "substitutions": 6,
+        "deletions": 1,
+        "insertions": 0,
+        "hits": 5
+      }
+    }
+  },
+  "multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "wer": {
+      "score": 1.0,
+      "overall": {
+        "substitutions": 3.25,
+        "deletions": 0.5,
+        "insertions": 0.25,
+        "hits": 2.0
+      }
+    }
+  },
+  "multiple_pred_multiple_ref": {
+    "total_items": 2,
+    "empty_items": 0,
+    "wer": {
+      "score": 1.2,
+      "overall": {
+        "substitutions": 3.0,
+        "deletions": 0.375,
+        "insertions": 0.625,
+        "hits": 2.375
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/tests/test_data/expected_outputs/test_jury.json b/tests/test_data/expected_outputs/test_jury.json
index 04af166..40dc864 100644
--- a/tests/test_data/expected_outputs/test_jury.json
+++ b/tests/test_data/expected_outputs/test_jury.json
@@ -355,9 +355,9 @@
     }
   },
   "evaluate_datasets_metric": {
-    "empty_items": 0,
     "total_items": 2,
-    "wer": 0.5833333333333334
+    "empty_items": 0,
+    "accuracy": 0.0
   },
   "evaluate_corpus": {
     "total_items": 2,