Spaces:
Running
Running
# -*- coding: utf-8 -*- | |
""" | |
# MANIFESTO ANALYSIS | |
""" | |
##IMPORTING LIBRARIES | |
import random | |
import matplotlib.pyplot as plt | |
import nltk | |
from nltk.tokenize import word_tokenize,sent_tokenize | |
from nltk.corpus import stopwords | |
from nltk.stem.porter import PorterStemmer | |
from nltk.stem import WordNetLemmatizer | |
from nltk.corpus import stopwords | |
from nltk.tokenize import word_tokenize | |
from nltk.probability import FreqDist | |
from cleantext import clean | |
import textract | |
import urllib.request | |
import nltk.corpus | |
from nltk.text import Text | |
import io | |
from io import StringIO,BytesIO | |
import sys | |
import pandas as pd | |
import cv2 | |
import re | |
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator | |
from textblob import TextBlob | |
from PIL import Image | |
import os | |
import gradio as gr | |
from zipfile import ZipFile | |
import contractions | |
import unidecode | |
nltk.download('stopwords') | |
nltk.download('punkt') | |
nltk.download('wordnet') | |
nltk.download('words') | |
"""## PARSING FILES""" | |
#def Parsing(parsed_text): | |
#parsed_text=parsed_text.name | |
#raw_party =parser.from_file(parsed_text) | |
# raw_party = raw_party['content'] | |
# return clean(raw_party) | |
def Parsing(parsed_text): | |
parsed_text=parsed_text.name | |
raw_party =textract.process(parsed_text, encoding='ascii',method='pdfminer') | |
return clean(raw_party) | |
#Added more stopwords to avoid irrelevant terms | |
stop_words = set(stopwords.words('english')) | |
stop_words.update('ask','much','thank','etc.', 'e', 'We', 'In', 'ed','pa', 'This','also', 'A', 'fu','To','5','ing', 'er', '2') | |
"""## PREPROCESSING""" | |
def clean_text(text): | |
''' | |
The function which returns clean text | |
''' | |
text = text.encode("ascii", errors="ignore").decode("ascii") # remove non-asciicharacters | |
text=unidecode.unidecode(text)# diacritics remove | |
text=contractions.fix(text) # contraction fix | |
text = re.sub(r"\n", " ", text) | |
text = re.sub(r"\n\n", " ", text) | |
text = re.sub(r"\t", " ", text) | |
text = re.sub(r"/ ", " ", text) | |
text = text.strip(" ") | |
text = re.sub(" +", " ", text).strip() # get rid of multiple spaces and replace with a single | |
text = [word for word in text.split() if word not in stop_words] | |
text = ' '.join(text) | |
return text | |
# text_Party=clean_text(raw_party) | |
def Preprocess(textParty): | |
''' | |
Removing special characters extra spaces | |
''' | |
text1Party = re.sub('[^A-Za-z0-9]+', ' ', textParty) | |
#Removing all stop words | |
pattern = re.compile(r'\b(' + r'|'.join(stopwords.words('english')) + r')\b\s*') | |
text2Party = pattern.sub('', text1Party) | |
# fdist_cong = FreqDist(word_tokens_cong) | |
return text2Party | |
''' | |
Using Concordance, you can see each time a word is used, along with its | |
immediate context. It can give you a peek into how a word is being used | |
at the sentence level and what words are used with it | |
''' | |
def concordance(text_Party,strng): | |
word_tokens_party = word_tokenize(text_Party) | |
moby = Text(word_tokens_party) | |
resultList = [] | |
for i in range(0,1): | |
save_stdout = sys.stdout | |
result = StringIO() | |
sys.stdout = result | |
moby.concordance(strng,lines=4,width=82) | |
sys.stdout = save_stdout | |
s=result.getvalue().splitlines() | |
return result.getvalue() | |
def normalize(d, target=1.0): | |
raw = sum(d.values()) | |
factor = target/raw | |
return {key:value*factor for key,value in d.items()} | |
def fDistance(text2Party): | |
''' | |
Most frequent words search | |
''' | |
word_tokens_party = word_tokenize(text2Party) #Tokenizing | |
fdistance = FreqDist(word_tokens_party).most_common(10) | |
mem={} | |
for x in fdistance: | |
mem[x[0]]=x[1] | |
return normalize(mem) | |
def fDistancePlot(text2Party,plotN=15): | |
''' | |
Most Frequent Words Visualization | |
''' | |
word_tokens_party = word_tokenize(text2Party) #Tokenizing | |
fdistance = FreqDist(word_tokens_party) | |
plt.title('Frequency Distribution') | |
plt.axis('off') | |
plt.figure(figsize=(4,3)) | |
fdistance.plot(plotN) | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img1 = Image.open(buf) | |
plt.clf() | |
return img1 | |
def DispersionPlot(textParty): | |
''' | |
Dispersion PLot | |
''' | |
word_tokens_party = word_tokenize(textParty) #Tokenizing | |
moby = Text(word_tokens_party) | |
fdistance = FreqDist(word_tokens_party) | |
word_Lst=[] | |
for x in range(5): | |
word_Lst.append(fdistance.most_common(6)[x][0]) | |
plt.axis('off') | |
plt.title('Dispersion Plot') | |
plt.figure(figsize=(4,3)) | |
moby.dispersion_plot(word_Lst) | |
plt.plot(color="#EF6D6D") | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img = Image.open(buf) | |
plt.clf() | |
return img | |
def getSubjectivity(text): | |
''' | |
Create a function to get the polarity | |
''' | |
return TextBlob(text).sentiment.subjectivity | |
def getPolarity(text): | |
''' | |
Create a function to get the polarity | |
''' | |
return TextBlob(text).sentiment.polarity | |
def getAnalysis(score): | |
if score < 0: | |
return 'Negative' | |
elif score == 0: | |
return 'Neutral' | |
else: | |
return 'Positive' | |
def Original_Image(path): | |
img= cv2.imread(path) | |
img= cv2.cvtColor(img, cv2.COLOR_BGR2RGB) | |
return img | |
def Image_Processed(path): | |
''' | |
Reading the image file | |
''' | |
img= cv2.imread(path) | |
img= cv2.cvtColor(img, cv2.COLOR_BGR2RGB) | |
#Thresholding | |
ret, bw_img = cv2.threshold(img, 124, 255, cv2.THRESH_BINARY) | |
return bw_img | |
def wordCloud(orgIm,mask_img,text_Party_pr,maxWord=2000,colorGener=True,contCol='white',bckColor='white'): | |
''' | |
Generating word cloud | |
''' | |
mask =mask_img | |
# Create and generate a word cloud image: | |
wordcloud = WordCloud(max_words=maxWord, background_color=bckColor, | |
mask=mask, | |
colormap='nipy_spectral_r', | |
contour_color=contCol, | |
width=800, height=800, | |
margin=2, | |
contour_width=3).generate(text_Party_pr) | |
# create coloring from image | |
if colorGener==True: | |
image_colors = ImageColorGenerator(orgIm) | |
plt.imshow(wordcloud.recolor(color_func= image_colors),interpolation="bilinear") | |
else: | |
plt.imshow(wordcloud) | |
plt.axis("off") | |
def word_cloud_generator(parsed_text_name,text_Party): | |
parsed=parsed_text_name.lower() | |
if 'bjp' in parsed: | |
orgImg=Original_Image('BJPMain.jpg') | |
bwImg=Image_Processed('BJPMain.jpg') | |
plt.figure(figsize=(15,12)) | |
wordCloud(orgImg,bwImg,text_Party,maxWord=3000,colorGener=True,contCol='white',bckColor='white') | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img1 = Image.open(buf) | |
plt.clf() | |
return img1 | |
elif 'congress' in parsed: | |
orgImg=Original_Image('congrsMain.jpg') | |
bwImg=Image_Processed('congrsMain.jpg') | |
plt.figure(figsize=(15,12)) | |
wordCloud(orgImg,bwImg,text_Party,maxWord=3000,colorGener=True) | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img2 = Image.open(buf) | |
plt.clf() | |
return img2 | |
elif 'aap' in parsed: | |
orgImg=Original_Image('AAPMain.jpg') | |
bwImg=Image_Processed('AAPMain.jpg') | |
plt.figure(figsize=(15,12)) | |
wordCloud(orgImg,bwImg,text_Party,maxWord=3000,colorGener=False,contCol='black') | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img3 = Image.open(buf) | |
plt.clf() | |
return img3 | |
else : | |
wordcloud = WordCloud(max_words=2000, background_color="white",mode="RGB").generate(text_Party) | |
plt.figure(figsize=(15,12)) | |
plt.imshow(wordcloud, interpolation="bilinear") | |
plt.axis("off") | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img4 = Image.open(buf) | |
plt.clf() | |
return img4 | |
''' | |
url = "http://library.bjp.org/jspui/bitstream/123456789/2988/1/BJP-Election-english-2019.pdf" | |
path_input = "./Bjp_Manifesto_2019.pdf" | |
urllib.request.urlretrieve(url, filename=path_input) | |
url="https://drive.google.com/uc?id=1BLCiy_BWilfVdrUH8kbO-44DJevwO5CG&export=download" | |
path_input = "./Aap_Manifesto_2019.pdf" | |
urllib.request.urlretrieve(url, filename=path_input) | |
url="https://drive.google.com/uc?id=1HVZvTtYntl0YKLnE0cwu0CvAIRhXOv60&export=download" | |
path_input = "./Congress_Manifesto_2019.pdf" | |
urllib.request.urlretrieve(url, filename=path_input) | |
''' | |
def analysis(Manifesto,Search): | |
raw_party = Parsing(Manifesto) | |
text_Party=clean_text(raw_party) | |
text_Party= Preprocess(text_Party) | |
df = pd.DataFrame(raw_party.split('\n'), columns=['Content']) | |
df['Subjectivity'] = df['Content'].apply(getSubjectivity) | |
df['Polarity'] = df['Content'].apply(getPolarity) | |
df['Analysis on Polarity'] = df['Polarity'].apply(getAnalysis) | |
df['Analysis on Subjectivity'] = df['Subjectivity'].apply(getAnalysis) | |
plt.title('Sentiment Analysis') | |
plt.xlabel('Sentiment') | |
plt.ylabel('Counts') | |
plt.figure(figsize=(4,3)) | |
df['Analysis on Polarity'].value_counts().plot(kind ='bar',color="#FF9F45") | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img1 = Image.open(buf) | |
plt.clf() | |
plt.figure(figsize=(4,3)) | |
df['Analysis on Subjectivity'].value_counts().plot(kind ='bar',color="#B667F1") | |
plt.tight_layout() | |
buf = BytesIO() | |
plt.savefig(buf) | |
buf.seek(0) | |
img2 = Image.open(buf) | |
plt.clf() | |
img3 = word_cloud_generator(Manifesto.name,text_Party) | |
fdist_Party=fDistance(text_Party) | |
img4=fDistancePlot(text_Party) | |
img5=DispersionPlot(text_Party) | |
searchRes=concordance(text_Party,Search) | |
searChRes=clean(searchRes) | |
searChRes=searchRes.replace(Search,"\u0332".join(Search)) | |
return searChRes,fdist_Party,img1,img2,img3,img4,img5 | |
Search_txt=gr.inputs.Textbox() | |
filePdf = gr.inputs.File() | |
text = gr.outputs.Textbox(label='SEARCHED OUTPUT') | |
mfw=gr.outputs.Label(label="Most Relevant Topics") | |
plot1=gr.outputs. Image(label='Sentiment Analysis') | |
plot2=gr.outputs.Image(label='Subjectivity Analysis') | |
plot3=gr.outputs.Image(label='Word Cloud') | |
plot4=gr.outputs.Image(label='Frequency Distribution') | |
plot5=gr.outputs.Image(label='Dispersion Plot') | |
io=gr.Interface(fn=analysis, inputs=[filePdf,Search_txt], outputs=[text,mfw,plot1,plot2,plot3,plot4,plot5], title='Manifesto Analysis',examples=[['manifestos/Bjp_Manifesto_2019.pdf','modi'],['manifestos/AAP_Manifesto_2019.pdf','delhi'],['manifestos/Congress_Manifesto_2019.pdf','safety']]) | |
io.launch(debug=True,share=False) | |
#,examples=[['./Bjp_Manifesto_2019.pdf','india'],['./AAP_Manifesto_2019.pdf',],['./Congress_Manifesto_2019.pdf',]] | |
#allow_screenshot=False, allow_flagging="never", | |
#examples=[['manifestos/Bjp_Manifesto_2019.pdf','modi'],['AAP_Manifesto_2019.pdf','delhi'],['manifestos/Congress_Manifesto_2019.pdf','safety']]) | |