Spaces:
Runtime error
Runtime error
import re | |
import string | |
from bs4 import BeautifulSoup | |
contraction_mapping = {"ain't": "is not", "aren't": "are not", "can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", | |
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", | |
"he'd": "he would", "he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", | |
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have", "I'm": "I am", | |
"I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have", "i'm": "i am", | |
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have", | |
"it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have", "mightn't": "might not", | |
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", | |
"needn't've": "need not have", "o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", | |
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", | |
"she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", | |
"so've": "so have", "so's": "so as", "this's": "this is", "that'd": "that would", "that'd've": "that would have", "that's": "that is", | |
"there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is", "they'd": "they would", | |
"they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", | |
"to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", | |
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", | |
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", | |
"where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", | |
"who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", | |
"would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would", | |
"y'all'd've": "you all would have", "y'all're": "you all are", "y'all've": "you all have", "you'd": "you would", "you'd've": "you would have", | |
"you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'u.s': 'america', 'e.g': 'for example'} | |
punct = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\', '•', '~', '@', '£', | |
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', | |
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', | |
'▒', ':', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', | |
'∙', ')', '↓', '、', '│', '(', '»', ',', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ] | |
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", | |
"`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', | |
'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!': ' '} | |
def clean_text(text): | |
'''Make text lowercase, remove text in square brackets, remove links, remove punctuation, | |
and remove words containing numbers.''' | |
text = re.sub(r'\:(.*?)\:', '', text) | |
text = str(text).lower() # Making Text Lowercase | |
text = re.sub('\[.*?\]', '', text) | |
# The next 2 lines remove html text | |
text = BeautifulSoup(text, 'lxml').get_text() | |
text = re.sub('https?://\S+|www\.\S+', '', text) | |
text = re.sub('<.*?>+', '', text) | |
text = re.sub('\n', '', text) | |
text = re.sub('\w*\d\w*', '', text) | |
# replacing everything with space except (a-z, A-Z, ".", "?", "!", ",", "'") | |
text = re.sub(r"[^a-zA-Z?.!,¿']+", " ", text) | |
return text | |
def clean_contractions(text, mapping): | |
'''Clean contraction using contraction mapping''' | |
specials = ["’", "‘", "´", "`"] | |
for s in specials: | |
text = text.replace(s, "'") | |
for word in mapping.keys(): | |
if "" + word + "" in text: | |
text = text.replace("" + word + "", "" + mapping[word] + "") | |
# Remove Punctuations | |
text = re.sub('[%s]' % re.escape(string.punctuation), '', text) | |
# creating a space between a word and the punctuation following it | |
# eg: "he is a boy." => "he is a boy ." | |
text = re.sub(r"([?.!,¿])", r" \1 ", text) | |
text = re.sub(r'[" "]+', " ", text) | |
return text | |
def clean_special_chars(text, punct, mapping): | |
'''Cleans special characters present (if any)''' | |
for p in mapping: | |
text = text.replace(p, mapping[p]) | |
for p in punct: | |
text = text.replace(p, f' {p} ') | |
specials = {'\u200b': ' ', '…': ' ... ', | |
'\ufeff': '', 'करना': '', 'है': ''} | |
for s in specials: | |
text = text.replace(s, specials[s]) | |
return text | |
def remove_space(text): | |
'''Removes awkward spaces''' | |
# Removes awkward spaces | |
text = text.strip() | |
text = text.split() | |
return " ".join(text) | |
def preprocess_text(text): | |
'''Cleaning and parsing the text.''' | |
text = clean_text(text) | |
text = clean_contractions(text, contraction_mapping) | |
text = clean_special_chars(text, punct, punct_mapping) | |
text = remove_space(text) | |
return text | |