Repository: jitsi/jiwer

Issue:
RemovePunctuation does not remove smart/curly quotes
RemovePunctuation does not remove the following:
‘
’
“
”

Discussion:

Patch:
diff --git a/README.md b/README.md
index 0b1633b..abf8839 100644
--- a/README.md
+++ b/README.md
@@ -211,9 +211,8 @@ print(jiwer.RemoveWhiteSpace(replace_by_space=True)(sentences))
 
 #### RemovePunctuation
 
-`jiwer.RemovePunctuation()` can be used to filter out punctuation. The punctuation characters are:
-
-``'!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'``
+`jiwer.RemovePunctuation()` can be used to filter out punctuation. The punctuation characters are defined as
+all unicode characters whose catogary name starts with `P`. See https://www.unicode.org/reports/tr44/#General_Category_Values. 
 
 Example:
 ```python
diff --git a/jiwer/transforms.py b/jiwer/transforms.py
index 1f717bb..024f82e 100644
--- a/jiwer/transforms.py
+++ b/jiwer/transforms.py
@@ -21,8 +21,10 @@ This file implements the building blocks for transforming a collection
 of input strings to the desired format in order to calculate the WER.
 """
 
+import sys
 import re
 import string
+import unicodedata
 
 from typing import Union, List, Mapping
 
@@ -189,9 +191,12 @@ class RemoveWhiteSpace(BaseRemoveTransform):
 
 class RemovePunctuation(BaseRemoveTransform):
     def __init__(self):
-        characters = [c for c in string.punctuation]
+        codepoints = range(sys.maxunicode + 1)
+        punctuation = set(
+            chr(i) for i in codepoints if unicodedata.category(chr(i)).startswith("P")
+        )
 
-        super().__init__(characters)
+        super().__init__(list(punctuation))
 
 
 class RemoveMultipleSpaces(AbstractTransform):


Test patch:
diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index 834c455..552e599 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -205,6 +205,16 @@ class TestRemovePunctuation(unittest.TestCase):
         cases = [
             (["this is an example!", "this is an example"]),
             (["hello. goodbye", "hello goodbye"]),
+            (["this sentence has no punctuation", "this sentence has no punctuation"]),
+        ]
+
+        _apply_test_on(self, RemovePunctuation(), cases)
+
+    def test_non_ascii_punctuation(self):
+        cases = [
+            (["word༆’'", "word"]),
+            (["‘no’", "no"]),
+            (["“yes”", "yes"]),
         ]
 
         _apply_test_on(self, RemovePunctuation(), cases)