|
from transformers import Tool |
|
|
|
import re,sys,unicodedata |
|
|
|
|
|
|
|
|
|
|
|
|
|
EMAIL_REGEX = re.compile(r"[\w\.-]+@[\w\.-]+") |
|
PHONE_REGEX = re.compile(r"[\+\(]?[1-9][0-9 .\-\(\)]{8,}[0-9]") |
|
NUMBERS_REGEX = re.compile(r"\d+") |
|
SPECIAL_CHARACTERS_REGEX = re.compile(r"[^A-Za-z0-9 ]+") |
|
EMOJI_REGEX = re.compile("[" |
|
u"\U0001F600-\U0001F64F" |
|
u"\U0001F300-\U0001F5FF" |
|
u"\U0001F680-\U0001F6FF" |
|
u"\U0001F1E0-\U0001F1FF" |
|
u"\U00002702-\U000027B0" |
|
u"\U000024C2-\U0001F251" |
|
"]+", flags=re.UNICODE) |
|
|
|
CURRENCIES = { |
|
"$": "USD", |
|
"zł": "PLN", |
|
"£": "GBP", |
|
"¥": "JPY", |
|
"฿": "THB", |
|
"₡": "CRC", |
|
"₦": "NGN", |
|
"₩": "KRW", |
|
"₪": "ILS", |
|
"₫": "VND", |
|
"€": "EUR", |
|
"₱": "PHP", |
|
"₲": "PYG", |
|
"₴": "UAH", |
|
"₹": "INR", |
|
} |
|
CURRENCY_REGEX = re.compile( |
|
"({})+".format("|".join(re.escape(c) for c in CURRENCIES.keys())) |
|
) |
|
|
|
|
|
|
|
class TextCleaner(object): |
|
"""TextCleaner: Class For Text Cleaning |
|
usage |
|
docx = TextCleaner() |
|
docx.text = "this is example@gmail.com and you can reach me at +380994777888 at 5pm#" |
|
|
|
""" |
|
def __init__(self, text=None): |
|
super(TextCleaner, self).__init__() |
|
self.text = text |
|
|
|
def __repr__(self): |
|
return "TextCleaner(text={})".format(self.text) |
|
|
|
def remove_emails(self): |
|
result = re.sub(EMAIL_REGEX,"",self.text) |
|
return result |
|
|
|
def remove_phone_numbers(self): |
|
result = re.sub(PHONE_REGEX,"",self.text) |
|
return result |
|
|
|
def remove_numbers(self): |
|
result = re.sub(NUMBERS_REGEX,"",self.text) |
|
return result |
|
|
|
def remove_special_characters(self): |
|
result = re.sub(SPECIAL_CHARACTERS_REGEX,"",self.text) |
|
return result |
|
|
|
def remove_emojis(self): |
|
result = re.sub(EMOJI_REGEX,"",self.text) |
|
return result |
|
|
|
def replace_emails(self,replace_with="<EMAIL>"): |
|
result = re.sub(EMAIL_REGEX,replace_with,self.text) |
|
return result |
|
|
|
def replace_phone_numbers(self,replace_with="<PHONENUMBER>"): |
|
result = re.sub(PHONE_REGEX,replace_with,self.text) |
|
return result |
|
|
|
def replace_numbers(self,replace_with="<NUMBER>"): |
|
result = re.sub(NUMBERS_REGEX,replace_with,self.text) |
|
return result |
|
|
|
def replace_special_characters(self,replace_with="<SPECIAL_CHAR>"): |
|
result = re.sub(SPECIAL_CHARACTERS_REGEX,replace_with,self.text) |
|
return result |
|
|
|
|
|
def clean_text(self,preserve=False): |
|
if preserve == False: |
|
email_result = re.sub(EMAIL_REGEX,"",self.text) |
|
phone_result = re.sub(PHONE_REGEX,"",email_result) |
|
number_result = re.sub(NUMBERS_REGEX,"",phone_result) |
|
emoji_result = re.sub(EMOJI_REGEX,"",number_result) |
|
special_char_result = re.sub(SPECIAL_CHARACTERS_REGEX,"",emoji_result) |
|
final_result = special_char_result.lower() |
|
|
|
else: |
|
special_char_result = re.sub(r'[^A-Za-z0-9@ ]+',"",self.text) |
|
email_result = re.sub(EMAIL_REGEX,"<EMAIL>",special_char_result) |
|
phone_result = re.sub(PHONE_REGEX,"<PHONENUMBER>",email_result) |
|
number_result = re.sub(NUMBERS_REGEX,"<NUMBERS>",phone_result) |
|
final_result = number_result.lower() |
|
|
|
return final_result |
|
|
|
|
|
class TextExtractor(TextCleaner): |
|
"""TextExtractor - Extract emails,numbers and phone numbers from text""" |
|
def __init__(self, text=None): |
|
super(TextExtractor, self).__init__() |
|
self.text = text |
|
|
|
def __repr__(self): |
|
return "TextExtractor(text={})".format(self.text) |
|
|
|
def extract_emails(self): |
|
match = re.findall(EMAIL_REGEX,self.text) |
|
return match |
|
|
|
def extract_phone_numbers(self): |
|
match = re.findall(PHONE_REGEX,self.text) |
|
return match |
|
|
|
def extract_numbers(self): |
|
match = re.findall(NUMBERS_REGEX,self.text) |
|
return match |
|
|
|
def extract_emojis(self): |
|
match = re.findall(EMOJI_REGEX,self.text) |
|
return match |
|
|
|
|
|
class TextifyTextTool(Tool): |
|
name = "token_counter" |
|
description = "This is a tool for cleaning text. It removes bad, unused characters." |
|
inputs = ["text"] |
|
outputs = ["text"] |
|
|
|
def __call__(self, text: str): |
|
docx = TextCleaner() |
|
docx.text = text |
|
docx.clean_text() |
|
text = docx.clean_text() |
|
print(docx) |
|
print("---") |
|
print(text) |
|
|
|
return text |
|
|
