Hindi_News_Summarizer / extractdata.py
d0r1h's picture
Create extractdata.py
66f6b0b
raw
history blame
1.05 kB
import re
import requests
from bs4 import BeautifulSoup
noise1 = re.compile(r"[([].*?[\)\]]\s+") # वर्ल्ड कप 2019 (World Cup 2019) --> वर्ल्ड कप 2019
noise2 = re.compile(r"\{.*?\}") # { googletag.display{ googletag.display(div-gpt-ad-1517823702248-0); });} }
noise3 = re.compile(r"[a-zA-Z]")
noise4 = re.compile(r"[\{()#@:%,_;&!=}\]]")
noise5 = re.compile(r'[\?\]]')
def extract_text(url):
data = requests.get(url)
soup = BeautifulSoup(data.content, "html.parser")
try:
vistaar = soup.find(class_ = "article-desc ul_styling")
vistaar = vistaar.text
except Exception as e:
print(f"Not able to fetch text {e}")
vistaar = vistaar.replace("विस्तार ", ' ')
vistaar = vistaar.replace("विज्ञापन", ' ')
vistaar = vistaar.replace("\n", ' ')
vistaar = re.sub('\xa0', ' ', vistaar)
vistaar = re.sub(noise2, ' ', vistaar)
vistaar = re.sub(noise3, ' ', vistaar)
vistaar = re.sub(noise4, ' ', vistaar)
vistaar = re.sub(' +', ' ', vistaar)
return vistaar