m3hrdadfi's picture
Hello gpt2-persian
21d29cb
raw
history blame
430 Bytes
import re
strange_double_quotes = [
"«",
"‹",
"»",
"›",
"„",
"“",
"‟",
"”",
"❝",
"❞",
"❮",
"❯",
"〝",
"〞",
"〟",
""",
]
strange_single_quotes = ["‘", "‛", "’", "❛", "❜", "`", "´", "‘", "’"]
DOUBLE_QUOTE_REGEX = re.compile("|".join(strange_double_quotes))
SINGLE_QUOTE_REGEX = re.compile("|".join(strange_single_quotes))