GitHub issue #2138: Add CER metric
State: closed | Created: 2021-03-29T15:52:27Z

Add Character Error Rate (CER) metric that is used in evaluation in ASR. I also have written unittests (hopefully thorough enough) but I'm not sure how to integrate them into the existed codebase.

```python
from cer import CER

cer = CER()

class TestCER(unittest.TestCase):
    def test_cer_case_senstive(self):
        refs = ['White House']
        preds = ['white house']
        # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.1818181818) < 1e-6)

    def test_cer_whitespace(self):
        refs = ['were wolf']
        preds = ['werewolf']
        # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.1111111) < 1e-6)

        refs = ['werewolf']
        preds = ['weae     wolf']
        # S = 1, D = 1, I = 0, N = 8, CER = 0.25
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.25) < 1e-6)

        # consecutive whitespaces case 1
        refs = ['were wolf']
        preds = ['were               wolf']
        # S = 0, D = 0, I = 0, N = 9, CER = 0
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.0) < 1e-6)

        # consecutive whitespaces case 2
        refs = ['were   wolf']
        preds = ['were               wolf']
        # S = 0, D = 0, I = 0, N = 9, CER = 0
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.0) < 1e-6)

    def test_cer_sub(self):
        refs = ['werewolf']
        preds = ['weaewolf']
        # S = 1, D = 0, I = 0, N = 8, CER = 0.125
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.125) < 1e-6)

    def test_cer_del(self):
        refs = ['werewolf']
        preds = ['wereawolf']
        # S = 0, D = 1, I = 0, N = 8, CER = 0.125
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.125) < 1e-6)

    def test_cer_insert(self):
        refs = ['werewolf']
        preds = ['wereolf']
        # S = 0, D = 0, I = 1, N = 8, CER = 0.125
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.125) < 1e-6)

    def test_cer_equal(self):
        refs = ['werewolf']
        char_error_rate = cer.compute(predictions=refs, references=refs)
        self.assertEqual(char_error_rate, 0.0)

    def test_cer_list_of_seqs(self):
        refs = ['werewolf', 'I am your father']
        char_error_rate = cer.compute(predictions=refs, references=refs)
        self.assertEqual(char_error_rate, 0.0)

        refs = ['werewolf', 'I am your father', 'doge']
        preds = ['werxwolf', 'I       am your father', 'doge']
        # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
        char_error_rate = cer.compute(predictions=preds, references=refs)
        self.assertTrue(abs(char_error_rate - 0.03571428) < 1e-6)

    def test_cer_unicode(self):
        ref = [u'我能吞下玻璃而不伤身体']
        pred = [u' 能吞虾玻璃而 不霜身体啦']
        # S = 3, D = 2, I = 0, N = 11
        # CER = 5 / 11
        char_error_rate = cer.compute(predictions=pred, references=ref)
        self.assertTrue(abs(char_error_rate - 0.4545454545) < 1e-6)

        ref = [u'我能吞', u'下玻璃而不伤身体']
        pred = [u'我    能 吞 下 玻 璃', u'而不伤身体']
        # S = 0, D = 5, I = 0, N = 11
        # CER = 5 / 11
        char_error_rate = cer.compute(predictions=pred, references=ref)
        self.assertTrue(abs(char_error_rate - 0.454545454545) < 1e-6)

        ref = [u'我能吞下玻璃而不伤身体']
        char_error_rate = cer.compute(predictions=ref, references=ref)
        self.assertFalse(char_error_rate, 0.0)

    def test_cer_empty(self):
        ref = ''
        pred = 'Hypothesis'
        with self.assertRaises(ValueError):
            char_error_rate = cer.compute(predictions=pred, references=ref)

if __name__ == '__main__':
    unittest.main()
```