Spaces:
Running
Running
| import re | |
| import regex | |
| import sys | |
| import textwrap | |
| from typing import Any, Dict, Optional | |
| punctuations = [ | |
| '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '.', | |
| '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', | |
| '`', '{', '|', '}', '~', '»', '«', '“', '”', "-", | |
| ] | |
| class Normalizer: | |
| """A general normalizer for every language""" | |
| _whitelist = r"[" + "\p{N}\p{L}\p{M}" + re.escape("".join(punctuations)) + "]+" | |
| _dictionary = {} | |
| def __init__( | |
| self, | |
| whitelist: str = None, | |
| dictionary: Dict[str, str] = None, | |
| ) -> None: | |
| self.whitelist = whitelist if whitelist and isinstance(whitelist, str) else self._whitelist | |
| self.dictionary = dictionary if dictionary and isinstance(dictionary, dict) else self._dictionary | |
| def chars_to_map(self, sentence: str) -> str: | |
| """Maps every character, words, and phrase into a proper one. | |
| Args: | |
| sentence (str): A piece of text. | |
| """ | |
| if not len(self.dictionary) > 0: | |
| return sentence | |
| pattern = "|".join(map(re.escape, self.dictionary.keys())) | |
| return re.sub(pattern, lambda m: self.dictionary[m.group()], str(sentence)) | |
| def chars_to_preserve( | |
| self, | |
| sentence: str, | |
| ) -> str: | |
| """Keeps specified characters from sentence | |
| Args: | |
| sentence (str): A piece of text. | |
| """ | |
| try: | |
| tokenized = regex.findall(self.whitelist, sentence) | |
| return " ".join(tokenized) | |
| except Exception as error: | |
| print( | |
| textwrap.dedent( | |
| f""" | |
| Bad characters range {self.whitelist}, | |
| {error} | |
| """ | |
| ) | |
| ) | |
| raise | |
| def text_level_normalizer(self, text: str) -> str: | |
| """A text level of normalization""" | |
| text = regex.sub(r"([" + re.escape("".join(punctuations)) + "])", r" \1 ", text) | |
| text = text.strip() | |
| return text | |
| def __call__( | |
| self, | |
| text: str, | |
| do_lowercase: Optional[bool] = False | |
| ) -> Any: | |
| """Normalization caller""" | |
| text = self.chars_to_map(text) | |
| text = self.chars_to_preserve(text) | |
| text = self.text_level_normalizer(text) | |
| text = re.sub(r"\s+", " ", text) | |
| if do_lowercase: | |
| text = text.lower() | |
| return text | |