all analysis

app.py

@@ -28,206 +28,271 @@ import seaborn as sns
 import plotly.express as px
 from wordcloud import WordCloud, STOPWORDS
 from textstat import flesch_reading_ease
+# import SessionState
+
 
 
 from nltk.sentiment.vader import SentimentIntensityAnalyzer
 sid = SentimentIntensityAnalyzer()
 
-# Settings
+def create_wordcloud(text):
+    st.header("Here is wordcloud..")
+    wc = WordCloud(width=width*100 , height=height*100 , background_color='white', colormap='prism', collocations = False).generate_from_text(text)
+    fig, ax = plt.subplots()
+    # fig, ax = plt.subplots(figsize=(width , height))
+    plt.imshow(wc,  interpolation='bilinear')
+    plt.axis('off')
+    st.pyplot(fig)
 
-width = st.sidebar.slider("plot width", 1, 25, 10)
-height = st.sidebar.slider("plot height", 1, 25, 4)
+# @st.cache(suppress_st_warning=True, allow_output_mutation=True)
+def get_input():
+    
+    text_dream= 'ex_dream.txt'
+    text_tryst= 'ex_tryst.txt'
+
+    with open(text_dream) as f:
+        dream = f.readlines()   
+    with open(text_tryst) as f:
+        tryst = f.readlines()     
+    
+    if 'x' not in st.session_state:
+        st.session_state['x'] = ' '
+    
+    if 'k' not in st.session_state:
+        st.session_state['k'] = 0
+    
+    if st.button('Example: I have a dream - M. King'):
+        st.session_state['x'] = ' '.join(dream)
+
+    if st.button('Example: Tryst with destiny - J. Nehru'):
+        st.session_state['x'] = ' '.join(tryst)
+    
+
+    em = st.empty()
+    if st.button('Clear'):
+        st.session_state['k']+=1
+        st.session_state['x'] = ' '
+
+    text = em.text_area("Paste your text or Click Example", value = st.session_state['x'] , key = st.session_state['k'], height=200, placeholder="Add here..")
+        
+    return text
+
+def create_ngram(text):
+    st.header("N-Gram Anaysis is >>") 
+    def plot_top_ngrams_barchart(text, n=2):
+        stop=set(stopwords.words('english'))
+        new= text.str.split()
+        new=new.values.tolist()
+        corpus=[word for i in new for word in i]
+
+        def _get_top_ngram(corpus, n=None):
+            vec = CountVectorizer(ngram_range=(n, n), stop_words=stop).fit(corpus)
+            bag_of_words = vec.transform(corpus)
+            sum_words = bag_of_words.sum(axis=0) 
+            words_freq = [(word, sum_words[0, idx]) 
+                        for word, idx in vec.vocabulary_.items()]
+            words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
+            return words_freq[:10]
+
+        top_n_bigrams=_get_top_ngram(text,n)[:10]
+        x,y=map(list,zip(*top_n_bigrams))
+        fig = px.bar(x=y,y=x, color=y)
+        fig.update_layout( yaxis=dict(autorange='reversed'))
+        fig.update_layout(autosize=False,width=width*100,height=height*100)
+        st.plotly_chart(fig)
+    
+    st.subheader(f"Unigram:")
+    plot_top_ngrams_barchart(pd.Series([text]), 1)
+    st.subheader(f"Bigram:")
+    plot_top_ngrams_barchart(pd.Series([text]), 2)
+    st.subheader(f"Trigram:")
+    plot_top_ngrams_barchart(pd.Series([text]), 3)
+
+# # Overall Sentiment
+def create_sentiment(text, tokenized_sent):
+    sentiment_dict = sid.polarity_scores(text)
+    st.header(f"Sentiment Analysis >>")
+    st.subheader(f"Overall Sentiment score  is = {sentiment_dict['compound']}")
+    # decide sentiment as positive, negative and neutral
+    if sentiment_dict['compound'] >= 0.05 :
+        st.subheader("Sentence Overall Rated As Positive")
+
+    elif sentiment_dict['compound'] <= - 0.05 :
+        st.subheader("Sentence Overall Rated As Negative")
+
+    else :
+        st.subheader("Sentence Overall Rated As Neutral")
+
+    # Temporal sentiment
+    st.subheader(f"Temporal Sentiment")
+
+    temporal_sentiment = pd.DataFrame(columns =['sentence', 'sentiment', 'len_sent'])
+    for sent in tokenized_sent:
+        sentiment_dict = sid.polarity_scores(sent)
+        temporal_sentiment = temporal_sentiment.append({'sentence' : sent,
+        'sentiment' :sentiment_dict['compound'],'len_sent' : len(sent.split())}, ignore_index=True)
+    temporal_sentiment['sentiment_stretch'] = (temporal_sentiment['sentiment'] * temporal_sentiment['len_sent']).astype(float)
+
+    fig = px.bar(temporal_sentiment, x=temporal_sentiment.index , y='sentiment',
+    hover_data=['sentence','sentiment','sentiment_stretch'], color= (temporal_sentiment['sentiment'] > 0),
+    color_discrete_map={True: 'green',False: 'red'})
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
 
-# Temp
-st.title("Text Disection : Analyze your text")
+    st.subheader(f"Temporal Sentiment Stretch")
+    fig = px.bar(temporal_sentiment, x=temporal_sentiment.index , y='sentiment_stretch',
+    hover_data=['sentence','sentiment','sentiment_stretch'], color= (temporal_sentiment['sentiment'] > 0),
+    color_discrete_map={True: 'green',False: 'red'})
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
+# # ner
+def nested_state(state):
+    # st.session_state['state_ner'] = state
+    st.session_state['nested_session'] = state
+
+def create_ner(text):
+    # st.session_state['state_ner'] = True
+    st.header(f"Named Entity Recognition >>")
+    st.subheader(f"Top Entities .. ")
+    doc=nlp(text)
+
+    ent = [X.label_ for X in doc.ents]
+    counter=Counter(ent)
+    count=counter.most_common()
+    x,y=map(list,zip(*count))
+    fig = px.bar(x=y,y=x, color=y)
+    fig.update_layout( yaxis=dict(autorange='reversed'))
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
 
+    st.subheader(f"What Are Those Entities .. ")
+    ent_type= st.selectbox("Select Named Entity :", x, on_change=nested_state(True))
+    ent_single = [X.text for X in doc.ents if X.label_ == ent_type]
+    ent_single=[x for x in ent_single]
+    counter=Counter(ent_single)
+    count=counter.most_common()
+    x,y=map(list,zip(*count))
+    fig = px.bar(x=y,y=x, color=y)
+    fig.update_layout( yaxis=dict(autorange='reversed'))
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
+
+    if st.button("Render NER"):
+        st.markdown(displacy.render(doc, style='ent'), unsafe_allow_html=True)
+
+
+# # pos tags
+def create_pos(text):
+    st.session_state['state_ner'] = True
+    st.header(f"Part of Speech >>")
+    st.subheader(f"Top POS ..")
+    # st.markdown(displacy.render(doc, style='dep'), unsafe_allow_html=True)
+    pos = nltk.pos_tag(tokenized_word)
+    pos=list(map(list,zip(*pos)))[1]
+    pos = [x for x in pos]
+    counter=Counter(pos)
+    count=counter.most_common()
+    x,y=map(list,zip(*count))
+    fig = px.bar(x=y,y=x, color=y)
+    fig.update_layout( yaxis=dict(autorange='reversed'))
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
+
+    st.subheader(f"What Are those POS .. ")
+    pos_type= st.selectbox("Select POS :", x)
+    pos_single = []
+    pos = nltk.pos_tag(tokenized_word)
+    for word,tag in pos:
+        if tag==pos_type:
+            pos_single.append(word)
+
+    pos_single=[x for x in pos_single]
+    counter=Counter(pos_single)
+    count=counter.most_common()
+    x,y=map(list,zip(*count))
+    fig = px.bar(x=y,y=x, color=y)
+    fig.update_layout( yaxis=dict(autorange='reversed'))
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
 
-# Input
-with st.container():
-    text_file = 'text.txt'
-    with open(text_file) as f:
-        text = f.readlines()    
-    default = ' '.join(text)
+# # Text Complexity
+def create_complexity(text, tokenized_sent):
+    st.header(f"Text Complexity >>")
+    st.caption(f"Higher scores indicate material that is easier to read,lower numbers mark harder-to-read passages:\
+    – 0-30 College\
+    – 50-60 High school\
+    – 60+ Fourth grade")
+    st.subheader(f"Flesch Reading Ease score  is = {flesch_reading_ease(text)}")
+
+    # Temporal sentiment
+    st.subheader(f"Temporal Complexity")
+
+    temporal_complexity= pd.DataFrame(columns =['sentence', 'complexity', 'len_sent'])
+    for sent in tokenized_sent:
+        complexity = flesch_reading_ease(sent)
+        temporal_complexity = temporal_complexity.append({'sentence' : sent,
+        'complexity' :complexity,'len_sent' : len(sent.split())}, ignore_index=True)
+    temporal_complexity['complexity_stretch'] = (temporal_complexity['complexity'] * temporal_complexity['len_sent']).astype(float)
+
+    fig = px.bar(temporal_complexity, x=temporal_complexity.index , y='complexity',
+    hover_data=['sentence','complexity','complexity_stretch'], color= (temporal_complexity['complexity'] > 30),
+    color_discrete_map={True: 'green',False: 'red'})
+    fig.update_layout(autosize=False,width=width*100,height=height*100)
+    st.plotly_chart(fig)
+
+if __name__ == '__main__':
+    m = st.markdown("""<style>div.stButton > button:first-child 
+    {background-color: #dbe6c4;}
+    </style>""", unsafe_allow_html=True)
+
+    st.title("Text Disection : Analyze your text")
+    st.sidebar.header("Adjust Plot Dimensions")
+    width = st.sidebar.slider("Plot Width", 1, 25, 10)
+    height = st.sidebar.slider("Plot Height", 1, 25, 7)
+
+
+    # Input
     st.header(f"Your Text please..")
-    text = st.text_area("Paste your text", default)
 
-# Baisc Info
-with st.container():
+    text = get_input()
     tokenized_sent=sent_tokenize(text)
     tokenized_word=word_tokenize(text)
-    fdist = FreqDist(tokenized_word)
-    st.header("Basic Information >>")
-    st.markdown(f"#### Total Sentences in the text = {len(tokenized_sent)}")
-    st.markdown(f"#### Total words in the text = {len(tokenized_word)}")
-    # st.markdown(f"##### Most common words = {fdist.most_common(5)}")
-
-# Word Cloud
-with st.container():
-    st.header("Here is wordcloud >>")
-    wc = WordCloud(background_color='black', colormap='RdYlGn').generate_from_text(text)
-    fig, ax = plt.subplots(figsize=(width, height))
-    plt.imshow(wc)
-    plt.axis('off')
-    st.pyplot(fig)
+    st.markdown(f"###### Total Sentences in the text = {len(tokenized_sent)}")
+    st.markdown(f"###### Total words in the text = {len(tokenized_word)}")
+    
+    
+    st.sidebar.title("Analysis Type")
+    analysis = st.sidebar.radio("Select Analysis", 
+    options = ['Wordcloud', 'N-Gram Analysis', 'Sentiment Analysis', 'Named Entity Recognition Analysis',
+    'Part Of Speech Analysis', 'Text Complexity Analysis','Keep Calm!'], index=6)
+
+
+    if st.button("Complete Analysis"):
+        create_wordcloud(text)
+        create_ngram(text)
+        create_sentiment(text, tokenized_sent)
+        create_ner(text)
+        create_pos(text)
+        create_complexity(text, tokenized_sent)
+        analysis = 'Keep Calm!'
+
+    if analysis == 'Wordcloud':
+        create_wordcloud(text)
+    if analysis == 'N-Gram Analysis':
+        create_ngram(text)
+    if analysis ==  'Sentiment Analysis':
+        create_sentiment(text, tokenized_sent)
+    if analysis == 'Named Entity Recognition Analysis':
+        create_ner(text)
+    if analysis ==  'Part Of Speech Analysis':
+        create_pos(text)
+    if analysis ==  'Text Complexity Analysis':
+        create_complexity(text, tokenized_sent)
+    if analysis == 'Keep Calm!':
+        st.image('nlp_meme.jpg')
+
+
+
+
+
 
-st.header("N-Gram Anaysis is >>") 
-def plot_top_ngrams_barchart(text, n=2):
-    stop=set(stopwords.words('english'))
-
-    new= text.str.split()
-    new=new.values.tolist()
-    corpus=[word for i in new for word in i]
-
-    def _get_top_ngram(corpus, n=None):
-        vec = CountVectorizer(ngram_range=(n, n), stop_words=stop).fit(corpus)
-        bag_of_words = vec.transform(corpus)
-        sum_words = bag_of_words.sum(axis=0) 
-        words_freq = [(word, sum_words[0, idx]) 
-                      for word, idx in vec.vocabulary_.items()]
-        words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
-        return words_freq[:10]
-
-    top_n_bigrams=_get_top_ngram(text,n)[:10]
-    x,y=map(list,zip(*top_n_bigrams))
-    # fig, ax = plt.subplots(figsize=(width, height))
-    # sns.barplot(x=y,y=x)
-    fig = px.bar(x=y,y=x, color=y)
-    fig.update_layout( yaxis=dict(autorange='reversed'))
-    st.plotly_chart(fig)
-    # st.pyplot()
-st.markdown(f'##### Unigram:')
-plot_top_ngrams_barchart(pd.Series([text]), 1)
-st.markdown(f'##### bigram:')
-plot_top_ngrams_barchart(pd.Series([text]), 2)
-st.markdown(f'##### Trigram:')
-plot_top_ngrams_barchart(pd.Series([text]), 3)
-
-
-# Overall Sentiment
-
-sentiment_dict = sid.polarity_scores(text)
-st.header(f"Sentiment Analysis >>")
-st.subheader(f"Overall Sentiment score  is = {sentiment_dict['compound']}")
-# decide sentiment as positive, negative and neutral
-if sentiment_dict['compound'] >= 0.05 :
-    st.markdown("** Sentence Overall Rated As Positive **")
-
-elif sentiment_dict['compound'] <= - 0.05 :
-    st.markdown("** Sentence Overall Rated As Negative **")
-
-else :
-    st.markdown("** Sentence Overall Rated As Neutral **")
-
-# Temporal sentiment
-st.subheader(f"Temporal Sentiment")
-
-temporal_sentiment = pd.DataFrame(columns =['sentence', 'sentiment', 'len_sent'])
-for sent in tokenized_sent:
-    sentiment_dict = sid.polarity_scores(sent)
-    temporal_sentiment = temporal_sentiment.append({'sentence' : sent,
-    'sentiment' :sentiment_dict['compound'],'len_sent' : len(sent.split())}, ignore_index=True)
-temporal_sentiment['sentiment_stretch'] = (temporal_sentiment['sentiment'] * temporal_sentiment['len_sent']).astype(float)
-
-fig = px.bar(temporal_sentiment, x=temporal_sentiment.index , y='sentiment',
-hover_data=['sentence','sentiment','sentiment_stretch'], color= (temporal_sentiment['sentiment'] > 0),
-color_discrete_map={True: 'green',False: 'red'})
-fig.update_layout(
-    autosize=False,
-    width=width*100,
-    height=height*100)
-st.plotly_chart(fig)
-
-st.subheader(f"Temporal Sentiment Stretch")
-fig = px.bar(temporal_sentiment, x=temporal_sentiment.index , y='sentiment_stretch',
-hover_data=['sentence','sentiment','sentiment_stretch'], color= (temporal_sentiment['sentiment'] > 0),
-color_discrete_map={True: 'green',False: 'red'})
-fig.update_layout(
-    autosize=False,
-    width=width*100,
-    height=height*100)
-st.plotly_chart(fig)
-
-# ner
-st.header(f"Named Entity Recognition >>")
-st.subheader(f"Top Entities .. ")
-doc=nlp(text)
-
-if st.button('Render NER ...'):
-    st.markdown(displacy.render(doc, style='ent'), unsafe_allow_html=True)
-
-ent = [X.label_ for X in doc.ents]
-counter=Counter(ent)
-count=counter.most_common()
-x,y=map(list,zip(*count))
-fig = px.bar(x=y,y=x, color=y)
-fig.update_layout( yaxis=dict(autorange='reversed'))
-st.plotly_chart(fig)
-
-st.subheader(f"What Are Those Entities .. ")
-ent_type= st.selectbox("Select Named Entity :", x)
-ent_single = [X.text for X in doc.ents if X.label_ == ent_type]
-ent_single=[x for x in ent_single]
-counter=Counter(ent_single)
-count=counter.most_common()
-x,y=map(list,zip(*count))
-fig = px.bar(x=y,y=x, color=y)
-fig.update_layout( yaxis=dict(autorange='reversed'))
-st.plotly_chart(fig)
-
-
-# pos tags
-st.header(f"Part of Speech >>")
-st.subheader(f"Top POS ..")
-# st.markdown(displacy.render(doc, style='dep'), unsafe_allow_html=True)
-pos = nltk.pos_tag(tokenized_word)
-pos=list(map(list,zip(*pos)))[1]
-pos = [x for x in pos]
-counter=Counter(pos)
-count=counter.most_common()
-x,y=map(list,zip(*count))
-fig = px.bar(x=y,y=x, color=y)
-fig.update_layout( yaxis=dict(autorange='reversed'))
-st.plotly_chart(fig)
-
-st.subheader(f"What Are those POS .. ")
-pos_type= st.selectbox("Select POS :", x)
-pos_single = []
-pos = nltk.pos_tag(tokenized_word)
-for word,tag in pos:
-    if tag==pos_type:
-        pos_single.append(word)
-
-pos_single=[x for x in pos_single]
-counter=Counter(pos_single)
-count=counter.most_common()
-x,y=map(list,zip(*count))
-fig = px.bar(x=y,y=x, color=y)
-fig.update_layout( yaxis=dict(autorange='reversed'))
-st.plotly_chart(fig)
-
-# Text Complexity
-st.header(f"Text Complexity >>")
-st.caption(f"Higher scores indicate material that is easier to read,lower numbers mark harder-to-read passages:\
-– 0-30 College\
-– 50-60 High school\
-– 60+ Fourth grade")
-st.subheader(f"Flesch Reading Ease score  is = {flesch_reading_ease(text)}")
-
-# Temporal sentiment
-st.subheader(f"Temporal Complexity")
-
-temporal_complexity= pd.DataFrame(columns =['sentence', 'complexity', 'len_sent'])
-for sent in tokenized_sent:
-    complexity = flesch_reading_ease(sent)
-    temporal_complexity = temporal_complexity.append({'sentence' : sent,
-    'complexity' :complexity,'len_sent' : len(sent.split())}, ignore_index=True)
-temporal_complexity['complexity_stretch'] = (temporal_complexity['complexity'] * temporal_complexity['len_sent']).astype(float)
-
-fig = px.bar(temporal_complexity, x=temporal_complexity.index , y='complexity',
-hover_data=['sentence','complexity','complexity_stretch'], color= (temporal_complexity['complexity'] > 30),
-color_discrete_map={True: 'green',False: 'red'})
-fig.update_layout(
-    autosize=False,
-    width=width*100,
-    height=height*100)
-st.plotly_chart(fig)

ex_tryst.txt

@@ -0,0 +1,47 @@
+"Long years ago... we made a tryst with destiny, and now the time comes when we shall redeem our pledge, not wholly or in full measure, but very substantially.
+
+At the stroke of the midnight hour, when the world sleeps, India will awake to life and freedom. A moment comes, which comes, but rarely in history, when we step out from the old to the new, when an age ends, and when the soul of a nation, long suppressed, finds utterance.
+
+It is fitting that at this solemn moment we take the pledge of dedication to the service of India and her people and to the still larger cause of humanity.
+
+At the dawn of history India started on her unending quest, and trackless centuries are filled with her striving and the grandeur of her success and her failures. Through good and ill fortune alike she has never lost sight of that quest or forgotten the ideals which gave her strength. We end today a period of ill fortune and India discovers herself again.
+
+The achievement we celebrate today is but a step, an opening of opportunity, to the greater triumphs and achievements that await us. Are we brave enough and wise enough to grasp this opportunity and accept the challenge of the future?
+
+Freedom and power bring responsibility. The responsibility rests upon this Assembly, a sovereign body representing the sovereign people of India. Before the birth of freedom we have endured all the pains of labour and our hearts are heavy with the memory of this sorrow. Some of those pains continue even now. Nevertheless, the past is over and it is the future that beckons to us now.
+
+That future is not one of ease or resting but of incessant striving so that we may fulfil the pledges we have so often taken and the one we shall take today. The service of India means the service of the millions who suffer. It means the ending of poverty and ignorance and disease and inequality of opportunity.
+
+The ambition of the greatest man of our generation has been to wipe every tear from every eye. That may be beyond us, but as long as there are tears and suffering, so long our work will not be over.
+
+And so we have to labour and to work, and work hard, to give reality to our dreams. Those dreams are for India, but they are also for the world, for all the nations and peoples are too closely knit together today for anyone of them to imagine that it can live apart.
+
+Peace has been said to be indivisible; so is freedom, so is prosperity now, and so also is disaster in this one world that can no longer be split into isolated fragments.
+
+To the people of India, whose representatives we are, we make an appeal to join us with faith and confidence in this great adventure. This is no time for petty and destructive criticism, no time for ill will or blaming others. We have to build the noble mansion of free India where all her children may dwell.
+
+The appointed day has come - the day appointed by destiny - and India stands forth again, after long slumber and struggle, awake, vital, free and independent. The past clings on to us still in some measure and we have to do much before we redeem the pledges we have so often taken. Yet the turning point is past, and history begins anew for us, the history which we shall live and act and others will write about.
+
+It is a fateful moment for us in India, for all Asia and for the world. A new star rises, the star of freedom in the east, a new hope comes into being, a vision long cherished materialises. May the star never set and that hope never be betrayed!
+
+We rejoice in that freedom, even though clouds surround us, and many of our people are sorrow-stricken and difficult problems encompass us. But freedom brings responsibilities and burdens and we have to face them in the spirit of a free and disciplined people.
+
+On this day our first thoughts go to the architect of this freedom, the father of our nation, who, embodying the old spirit of India, held aloft the torch of freedom and lighted up the darkness that surrounded us.
+
+We have often been unworthy followers of his and have strayed from his message, but not only we but succeeding generations will remember this message and bear the imprint in their hearts of this great son of India, magnificent in his faith and strength and courage and humility. We shall never allow that torch of freedom to be blown out, however high the wind or stormy the tempest.
+
+Our next thoughts must be of the unknown volunteers and soldiers of freedom who, without praise or reward, have served India even unto death.
+
+We think also of our brothers and sisters who have been cut off from us by political boundaries and who unhappily cannot share at present in the freedom that has come. They are of us and will remain of us whatever may happen, and we shall be sharers in their good and ill fortune alike.
+
+The future beckons to us. Whither do we go and what shall be our endeavour? To bring freedom and opportunity to the common man, to the peasants and workers of India; to fight and end poverty and ignorance and disease; to build up a prosperous, democratic and progressive nation, and to create social, economic and political institutions which will ensure justice and fullness of life to every man and woman.
+
+We have hard work ahead. There is no resting for any one of us till we redeem our pledge in full, till we make all the people of India what destiny intended them to be.
+
+We are citizens of a great country, on the verge of bold advance, and we have to live up to that high standard. All of us, to whatever religion we may belong, are equally the children of India with equal rights, privileges and obligations. We cannot encourage communism or narrow-mindedness, for no nation can be great whose people are narrow in thought or in action.
+
+To the nations and people of the world we send greetings and pledge ourselves to cooperate with them in furthering peace, freedom and democracy.
+
+And to India, our much-loved motherland, the ancient, the eternal and the ever-new, we pay our reverent homage and we bind ourselves afresh to her service.
+
+Jai Hind."