Repository: jitsi/jiwer

Issue:
utterance ref cannot be empty ?
Hello,
is there a good reason why an utterance ref is required to be non-empty ?
https://github.com/jitsi/jiwer/blob/9db6e4649dfff1e91de5640e224ea51de01b0a50/jiwer/process.py#L158C1-L159C69

IMHO, i'd expect that it can be empty (sclite behavior). 
It is a valid situation, if the utterance in test set contains just silence, it's reference is empty,
and the ASR system should produce an empty string and not hallucinate any symbol.

I hacked it accordingly here:
https://github.com/KarelVesely84/jiwer/tree/allow_empty_ref

Best regards
Karel Vesely




Discussion:

Patch:
diff --git a/README.md b/README.md
index 3db8e07..be94f9f 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,39 @@ hypothesis = "hello duck"
 error = wer(reference, hypothesis)
 ```
 
+
+## A note on empty references
+
+There is undefined behaviour when you apply an empty reference and hypothesis pair
+to the WER formula, as you get a division by zero.
+
+As of version 4.0, `jiwer` defines the behaviour as follows:
+
+```python3
+import jiwer
+
+# when ref and hyp are both empty, there is no error as
+# an ASR system correctly predicted silence/non-speech.
+assert jiwer.wer('', '') == 0 
+assert jiwer.mer('', '') == 0
+assert jiwer.wip('', '') == 1
+assert jiwer.wil('', '') == 0
+
+assert jiwer.cer('', '') == 0
+```
+
+When the hypothesis is non-empty, every word or character counts as an insertion:
+```python3
+import jiwer
+
+assert jiwer.wer('', 'silence') == 1
+assert jiwer.wer('', 'peaceful silence') == 2
+assert jiwer.process_words('', 'now defined behaviour').insertions == 3
+
+assert jiwer.cer('', 'a') == 1
+assert jiwer.cer('', 'abcde') == 5
+```
+
 ## Licence
 
 The jiwer package is released under the `Apache License, Version 2.0` licence by [8x8](https://www.8x8.com/).
diff --git a/docs/index.md b/docs/index.md
index 72e1e93..16203cb 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -26,6 +26,34 @@ Or, if you prefer old-fashioned pip and you're using Python >= `3.8`:
 $ pip install jiwer
 ```
 
+## A note on empty references
 
+There is undefined behaviour when you apply an empty reference and hypothesis pair
+to the WER formula, as you get a division by zero.
 
+As of version 4.0, `jiwer` defines the behaviour as follows:
 
+```python3
+import jiwer
+
+# when ref and hyp are both empty, there is no error as
+# an ASR system correctly predicted silence/non-speech.
+assert jiwer.wer('', '') == 0 
+assert jiwer.mer('', '') == 0
+assert jiwer.wip('', '') == 1
+assert jiwer.wil('', '') == 0
+
+assert jiwer.cer('', '') == 0
+```
+
+When the hypothesis is non-empty, every word or character counts as an insertion:
+```python3
+import jiwer
+
+assert jiwer.wer('', 'silence') == 1
+assert jiwer.wer('', 'peaceful silence') == 2
+assert jiwer.process_words('', 'now defined behaviour').insertions == 3
+
+assert jiwer.cer('', 'a') == 1
+assert jiwer.cer('', 'abcde') == 5
+```
diff --git a/src/jiwer/alignment.py b/src/jiwer/alignment.py
index 3ce4c22..b22f789 100644
--- a/src/jiwer/alignment.py
+++ b/src/jiwer/alignment.py
@@ -103,7 +103,9 @@ def visualize_alignment(
 
     final_str = ""
     for idx, (gt, hp, chunks) in enumerate(zip(references, hypothesis, alignment)):
-        if skip_correct and len(chunks) == 1 and chunks[0].type == "equal":
+        if skip_correct and (
+            len(chunks) == 0 or (len(chunks) == 1 and chunks[0].type == "equal")
+        ):
             continue
 
         final_str += f"sentence {idx+1}\n"
diff --git a/src/jiwer/process.py b/src/jiwer/process.py
index 38a27ac..69936dd 100644
--- a/src/jiwer/process.py
+++ b/src/jiwer/process.py
@@ -161,8 +161,6 @@ def process_words(
         reference = [reference]
     if isinstance(hypothesis, str):
         hypothesis = [hypothesis]
-    if any(len(t) == 0 for t in reference):
-        raise ValueError("one or more references are empty strings")
 
     # pre-process reference and hypothesis by applying transforms
     ref_transformed = _apply_transform(
@@ -237,13 +235,29 @@ def process_words(
     # Compute all measures
     S, D, I, H = num_substitutions, num_deletions, num_insertions, num_hits
 
-    wer = float(S + D + I) / float(H + S + D)
-    mer = float(S + D + I) / float(H + S + D + I)
-    wip = (
-        (float(H) / num_rf_words) * (float(H) / num_hp_words)
-        if num_hp_words >= 1
-        else 0
-    )
+    # special edge-case for empty references
+    if num_rf_words == 0:
+        wer = num_insertions
+
+        # if the reference was silence and this is correctly predicted,
+        # there is no error and all information is preserved
+        if num_hp_words == 0:
+            mer = 0
+            wip = 1
+        else:
+            mer = 1
+            wip = 0
+
+    else:
+        wer = float(S + D + I) / float(H + S + D)
+        mer = float(S + D + I) / float(H + S + D + I)
+
+        # there is an edge-case when hypothesis is empty
+        if num_hp_words >= 1:
+            wip = (float(H) / num_rf_words) * (float(H) / num_hp_words)
+        else:
+            wip = 0
+
     wil = 1 - wip
 
     # return all output
@@ -329,8 +343,6 @@ def process_characters(
         (CharacterOutput): The processed reference and hypothesis sentences.
 
     """
-    # make sure the transforms end with tr.ReduceToListOfListOfChars(),
-
     # it's the same as word processing, just every word is of length 1
     result = process_words(
         reference, hypothesis, reference_transform, hypothesis_transform
@@ -362,27 +374,19 @@ def _apply_transform(
     transformed_sentence = transform(sentence)
 
     # Validate the output is a list containing lists of strings
-    if is_reference:
-        if not _is_list_of_list_of_strings(
-            transformed_sentence, require_non_empty_lists=True
-        ):
-            raise ValueError(
-                "After applying the transformation, each reference should be a "
-                "non-empty list of strings, with each string being a single word."
-            )
-    else:
-        if not _is_list_of_list_of_strings(
-            transformed_sentence, require_non_empty_lists=False
-        ):
-            raise ValueError(
-                "After applying the transformation, each hypothesis should be a "
-                "list of strings, with each string being a single word."
-            )
+    if not _is_list_of_list_of_strings(transformed_sentence):
+        raise ValueError(
+            "After applying the transformation, each "
+            f"{'reference' if is_reference else 'hypothesis'} should be a "
+            "list of strings, with each string being a single word or character."
+            "Please ensure the given transformation reduces the input "
+            "to a list of list strings."
+        )
 
     return transformed_sentence
 
 
-def _is_list_of_list_of_strings(x: Any, require_non_empty_lists: bool):
+def _is_list_of_list_of_strings(x: Any):
     if not isinstance(x, list):
         return False
 
@@ -390,9 +394,6 @@ def _is_list_of_list_of_strings(x: Any, require_non_empty_lists: bool):
         if not isinstance(e, list):
             return False
 
-        if require_non_empty_lists and len(e) == 0:
-            return False
-
         if not all([isinstance(s, str) for s in e]):
             return False
 
diff --git a/src/jiwer/transformations.py b/src/jiwer/transformations.py
index 37fcbb1..f6ee13b 100644
--- a/src/jiwer/transformations.py
+++ b/src/jiwer/transformations.py
@@ -52,6 +52,16 @@ Then each string is transformed into a list with lists of strings, where each st
 is a single word.
 """
 
+wer_allow_empty_ref = tr.Compose(
+    [
+        tr.RemoveMultipleSpaces(),
+        tr.Strip(),
+        tr.SubstituteWords({"": "|silence|"}),
+        tr.ReduceToListOfListOfWords(),
+    ]
+)
+
+
 wer_contiguous = tr.Compose(
     [
         tr.RemoveMultipleSpaces(),
diff --git a/src/jiwer/transforms.py b/src/jiwer/transforms.py
index 4f8bbe0..3932a3a 100644
--- a/src/jiwer/transforms.py
+++ b/src/jiwer/transforms.py
@@ -183,7 +183,6 @@ class ReduceToListOfListOfWords(AbstractTransform):
 
         for sentence in inp:
             list_of_words = self.process_string(sentence)[0]
-
             sentence_collection.append(list_of_words)
 
         if len(sentence_collection) == 0:


Test patch:
diff --git a/tests/test_alignment.py b/tests/test_alignment.py
index e1f52cd..c3a56c7 100644
--- a/tests/test_alignment.py
+++ b/tests/test_alignment.py
@@ -123,6 +123,26 @@ class TestAlignmentVisualizationWords(unittest.TestCase):
         )
         self.assertEqual(alignment, correct_alignment)
 
+    def test_empty_ref(self):
+        correct_alignment = ""
+        alignment = jiwer.visualize_alignment(
+            jiwer.process_words("", ""),
+            show_measures=False,
+        )
+        self.assertEqual(alignment, correct_alignment)
+
+    def test_empty_ref_with_hyp_deletion(self):
+        correct_alignment = """sentence 1
+REF: ********
+HYP: inserted
+            I
+"""
+        alignment = jiwer.visualize_alignment(
+            jiwer.process_words("", "inserted"),
+            show_measures=False,
+        )
+        self.assertEqual(alignment, correct_alignment)
+
 
 class TestAlignmentVisualizationCharacters(unittest.TestCase):
     def test_insertion(self):
@@ -223,3 +243,23 @@ class TestAlignmentVisualizationCharacters(unittest.TestCase):
             show_measures=False,
         )
         self.assertEqual(alignment, correct_alignment)
+
+    def test_empty_ref(self):
+        correct_alignment = ""
+        alignment = jiwer.visualize_alignment(
+            jiwer.process_characters("", ""),
+            show_measures=False,
+        )
+        self.assertEqual(alignment, correct_alignment)
+
+    def test_empty_ref_with_hyp_deletion(self):
+        correct_alignment = """sentence 1
+REF: ********
+HYP: inserted
+     IIIIIIII
+"""
+        alignment = jiwer.visualize_alignment(
+            jiwer.process_characters("", "inserted"),
+            show_measures=False,
+        )
+        self.assertEqual(alignment, correct_alignment)
diff --git a/tests/test_cer.py b/tests/test_cer.py
index 99ddb50..dcf2fa9 100644
--- a/tests/test_cer.py
+++ b/tests/test_cer.py
@@ -49,12 +49,6 @@ class TestCERInputMethods(unittest.TestCase):
 
         self.assertRaises(ValueError, callback)
 
-    def test_fail_on_empty_reference(self):
-        def callback():
-            jiwer.cer("", "test")
-
-        self.assertRaises(ValueError, callback)
-
     def test_known_values(self):
         # Taken from the "From WER and RIL to MER and WIL" paper, for link see README.md
         cases = [
diff --git a/tests/test_empty_ref.py b/tests/test_empty_ref.py
new file mode 100644
index 0000000..278a966
--- /dev/null
+++ b/tests/test_empty_ref.py
@@ -0,0 +1,108 @@
+import pytest
+import jiwer
+
+
+def test_empty_ref_empty_hyp():
+    for i in range(10):
+        if i == 0:
+            ref = ""
+            hyp = ""
+        else:
+            ref = [""] * i
+            hyp = [""] * i
+
+        out = jiwer.process_words(reference=ref, hypothesis=hyp)
+
+        assert out.hits == 0
+        assert out.deletions == 0
+        assert out.insertions == 0
+        assert out.substitutions == 0
+
+        assert out.wer == 0
+        assert out.mer == 0
+        assert out.wip == 1
+        assert out.wil == 0
+
+
+def test_empty_ref():
+    out = jiwer.process_words(reference="", hypothesis="hello")
+
+    assert out.insertions == 1
+
+    assert out.wer == 1
+    assert out.mer == 1
+    assert out.wip == 0
+    assert out.wil == 1
+
+
+def test_empty_ref_more():
+    for i in range(2, 11):
+        out = jiwer.process_words(
+            reference="", hypothesis=" ".join(str(j) for j in range(i))
+        )
+
+        assert out.insertions == i
+
+        assert out.wer == i
+        assert out.mer == 1
+        assert out.wip == 0
+        assert out.wil == 1
+
+
+def test_empty_hyp():
+    out = jiwer.process_words(reference="hello", hypothesis="")
+
+    assert out.deletions == 1
+
+    assert out.wer == 1
+    assert out.mer == 1
+    assert out.wip == 0
+    assert out.wil == 1
+
+
+def test_multiple_one_empty_ref():
+    out = jiwer.process_words(
+        reference=["hello world", ""], hypothesis=["hello world", "done"]
+    )
+
+    assert out.insertions == 1
+    assert out.wer == pytest.approx(1 / 2)
+    assert out.mer == pytest.approx(1 / 3)
+    assert out.wil == pytest.approx(1 / 3)
+    assert out.wip == pytest.approx(2 / 3)
+
+
+def test_one_empty_ref_and_hyp():
+    out_without = jiwer.process_words(
+        reference=["hello world"], hypothesis=["hello world"]
+    )
+    out_with = jiwer.process_words(
+        reference=["hello world", ""], hypothesis=["hello world", ""]
+    )
+
+    assert out_without.hits == out_with.hits
+    assert out_without.substitutions == out_with.substitutions
+    assert out_without.insertions == out_with.insertions
+    assert out_without.deletions == out_with.deletions
+
+    assert out_without.wer == out_with.wer
+    assert out_without.mer == out_with.mer
+    assert out_without.wip == out_with.wip
+    assert out_without.wil == out_with.wil
+
+
+def test_cer_empty_ref():
+    out = jiwer.process_characters("", "")
+    assert out.cer == 0
+
+    out = jiwer.process_characters("", "a")
+    assert out.cer == 1
+
+    out = jiwer.process_characters("", "abd")
+    assert out.cer == 3
+
+    out = jiwer.process_characters("a", "")
+    assert out.cer == 1
+
+    out = jiwer.process_characters("abc", "")
+    assert out.cer == 1
diff --git a/tests/test_measures.py b/tests/test_measures.py
index 50fc744..a9f05c6 100644
--- a/tests/test_measures.py
+++ b/tests/test_measures.py
@@ -1,7 +1,5 @@
 import unittest
-
 import pytest
-
 import jiwer
 
 
@@ -110,20 +108,6 @@ class TestMeasuresContiguousSentencesTransform(unittest.TestCase):
 
         assert_dict_almost_equal(self, x_dict, y_dict, delta=1e-9)
 
-    def test_fail_on_empty_reference(self):
-        for method in [
-            jiwer.wer,
-            jiwer.wil,
-            jiwer.wip,
-            jiwer.mer,
-            jiwer.compute_measures,
-        ]:
-
-            def callback():
-                method("", "test")
-
-            self.assertRaises(ValueError, callback)
-
     def test_known_values(self):
         # Taken from the "From WER and RIL to MER and WIL" paper, for link see README.md
         cases = [
@@ -240,21 +224,6 @@ class TestMeasuresDefaultTransform(unittest.TestCase):
 
             self.assertRaises(ValueError, callback)
 
-    def test_fail_on_empty_reference(self):
-        for method in [
-            jiwer.process_words,
-            jiwer.wer,
-            jiwer.wil,
-            jiwer.wip,
-            jiwer.mer,
-            jiwer.compute_measures,
-        ]:
-
-            def callback():
-                method("", "test")
-
-            self.assertRaises(ValueError, callback)
-
     def test_known_values(self):
         # Taken from the "From WER and RIL to MER and WIL" paper, for link see README.md
         cases = [