Upload 4 files

Metadata.py

@@ -0,0 +1,317 @@
+import streamlit as st
+import streamlit.components.v1 as stc
+import pandas as pd
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from PIL import Image
+import exifread  # Extracts Meta data of images
+import os
+from datetime import datetime
+
+import mutagen # Extracts Meta data of Audio
+from PIL.ExifTags import TAGS, GPSTAGS
+import base64
+import time
+from PyPDF2 import PdfReader
+timestr = time.strftime("%Y%m%d-%H%M%S")
+
+import sqlite3
+
+details = """
+Metadata is defined as the data providing information about one or more aspects of the data; it is used to summarize basic information about data which can make tracking and working with specific data easier
+"""
+
+HTML_BANNER = """
+    <div style="background-color:violet;padding:10px;border-radius:10px">
+    <h1 style="color:white;text-align:center;">MetaData Extractor App </h1>
+    </div>
+    """
+
+def file_download(data):
+    csv_file= data.to_csv()
+    b64=base64.b64encode(csv_file.encode()).decode()
+    new_filename="result_{}.csv".format(timestr)
+    st.markdown('### 🗃️ Download csv file ')
+    href=f'<a href="data:file/csv;base64,{b64}" download="{new_filename}"> Click Here! </a>'
+    st.markdown(href, unsafe_allow_html=True)
+
+conn=sqlite3.connect('data.db')
+c=conn.cursor()
+    
+def create_filestable():
+    c.execute('CREATE TABLE IF NOT EXISTS filestable(filename TEXT,filetype TEXT,filesize TEXT,uploadDate TIMESTAMP)')
+
+def add_file_details():
+    c.execute('INSERT INTO filestable(filename, filetype, filesize, uploadDate) VALUES (?, ?, ?, ?)', (filename, filetype, filesize, uploadDate))
+    conn.commit()
+
+def view_all_data():
+	c.execute('SELECT * FROM filestable')
+	data = c.fetchall()
+	return data
+
+
+def load_image(file):
+    img = Image.open(file)
+    return img
+
+def get_readable_time(time):
+    return datetime.fromtimestamp(time).strftime('%Y-%m-%d-%H:%M')
+
+
+def get_exif(filename):
+    exif = Image.open(filename).getexif()
+
+    if exif is not None  and isinstance(exif, dict):
+        for key, value in exif.items():
+            name = TAGS.get(key, value)
+            exif[name] = exif.pop(key)
+
+        if 'GPSInfo' in exif:
+            for key in exif['GPSInfo'].keys():
+                name = GPSTAGS.get(key,key)
+                exif['GPSInfo'][name] = exif['GPSInfo'].pop(key)
+    return exif
+
+
+def metadata():
+    # st.title('Meta-Data Extractor App')
+    stc.html(HTML_BANNER)
+    menu=['Home','Image','Audio','Document_Files','Analytics']
+    choice=st.sidebar.selectbox('Menu',menu)
+    create_filestable()
+    if choice=='Home':
+        st.image(load_image('extraction_process.png'))
+        st.write(details)
+        col1, col2, col3 = st.columns(3)
+        with col1:
+            with st.expander("Get Image Metadata 📷"):
+                st.info("Image Metadata")
+                st.markdown("📷")
+                st.text("Upload JPEG,JPG,PNG Images")
+
+        with col2:
+            with st.expander("Get Audio Metadata 🔉"):
+                st.info("Audio Metadata")
+                st.markdown("🔉")
+                st.text("Upload Mp3,Ogg")
+
+        with col3:
+            with st.expander("Get Document Metadata 📄📁"):
+                st.info("Document Files Metadata")
+                st.markdown("📄📁")
+                st.text("Upload PDF,Docx")
+
+    elif choice=='Image':
+        st.subheader('Image MetaData Extractor')
+        image_file = st.file_uploader("Upload Image", type=["png", "jpg", "jpeg"])
+        if image_file is not None:
+            with st.expander('File Stats'):
+                file_details={'Filename':image_file.name,
+                              'Filesize':image_file.size,
+                              'Filetype':image_file.type}
+
+                statinfo=os.stat(image_file.readable())
+                statdetails={
+                    'Accessed Time': get_readable_time(statinfo.st_atime),
+                    'Creation Time':get_readable_time(statinfo.st_ctime),
+                    'Modified Time':get_readable_time(statinfo.st_mtime)}
+                full_details={
+                    'Filename':image_file.name,
+                     'Filesize':image_file.size,
+                    'Filetype':image_file.type,
+                    'Accessed Time': get_readable_time(statinfo.st_atime),
+                    'Creation Time':get_readable_time(statinfo.st_ctime),
+                    'Modified Time':get_readable_time(statinfo.st_mtime)
+                            }
+                # st.write(full_details)
+                file_details_df = pd.DataFrame(
+                    list(full_details.items()), columns=["Meta Tags", "Value"]
+                )
+                st.dataframe(file_details_df)
+            c1, c2 = st.columns(2)
+            with c1:
+                with st.expander("View Image"):
+                    img = load_image(image_file)
+                    st.image(img,width=250)
+            with c2:
+                with st.expander("Default(JPEG)"):
+                    st.info("Using PILLOW")
+                    img = load_image(image_file)
+                    img_details = {
+                        "format": img.format,
+                        "format_desc": img.format_description,
+                        "filename": img.filename,
+                        "size": img.size,
+                        "height": img.height,
+                        "width": img.width,
+                        "info": img.info,
+                    }
+                    df_img_details = pd.DataFrame(
+                        list(img_details.items()), columns=["Meta Tags", "Value"]
+                    )
+                    st.dataframe(df_img_details)
+
+            c3,c4=st.columns(2)
+            with c3:
+                with st.expander('Using ExifRead Tool'):
+                    meta_data=exifread.process_file(image_file)
+                    # st.write(meta_data)
+                    meta_data_df=pd.DataFrame(
+                        list(meta_data.items()),columns=['Meta Data','Values'])
+                    st.dataframe(meta_data_df)
+            with c4:
+                with st.expander('Image geo Coordinates'):
+                    img_gps_details=get_exif(image_file)
+                    latitude = img_gps_details.get('GPSLatitude')
+                    longitude = img_gps_details.get('GPSLongitude')       
+                    try:
+                        gps_info = img_gps_details
+                        lat=latitude
+                        long=longitude
+                    except:
+                        gps_info = "None Found"
+                    st.write(gps_info)
+                    st.write(lat)
+                    st.write(long)
+
+            add_file_details(img.filename,img.format,img.size,datetime.now())
+            with st.expander('Download Results'):
+                final_df=pd.concat([file_details_df,df_img_details,meta_data_df])
+                st.dataframe(final_df)
+                file_download(final_df)
+
+
+    elif choice=='Audio':
+        st.subheader('Audio MetaData Extractor')
+        audio_file = st.file_uploader("Upload Audio", type=["mp3", "ogg"])
+
+        if audio_file is not None:
+
+            col1, col2 = st.columns(2)
+
+            with col1:
+                st.audio(audio_file.read())
+
+            with col2:
+                with st.expander("File Stats"):
+                    file_details = {
+                        "FileName": audio_file.name,
+                        "FileSize": audio_file.size,
+                        "FileType": audio_file.type,
+                    }
+                    add_file_details(audio_file.name,audio_file.type,audio_file.size,datetime.now())
+
+                    st.write(file_details)
+
+                    statinfo = os.stat(audio_file.readable())
+                    stats_details = {
+                        "Accessed_Time": get_readable_time(statinfo.st_atime),
+                        "Creation_Time": get_readable_time(statinfo.st_ctime),
+                        "Modified_Time": get_readable_time(statinfo.st_mtime),
+                    }
+                    st.write(stats_details)
+
+                    file_details_combined = {
+                        "FileName": audio_file.name,
+                        "FileSize": audio_file.size,
+                        "FileType": audio_file.type,
+                        "Accessed_Time": get_readable_time(statinfo.st_atime),
+                        "Creation_Time": get_readable_time(statinfo.st_ctime),
+                        "Modified_Time": get_readable_time(statinfo.st_mtime),
+                    }
+
+                    df_file_details = pd.DataFrame(
+                        list(file_details_combined.items()),
+                        columns=["Meta Tags", "Value"],
+                    )
+                    st.dataframe(df_file_details)
+
+                with st.expander('Metadata using Mutagen'):
+                    meta_data=mutagen.File(audio_file)
+                    meta_data_dict={str(key):str(value) for key,value in meta_data.items()}
+                    meta_data_audio_df=pd.DataFrame(
+                        list(meta_data_dict.items()),columns=['Tag','Values'])
+                    st.dataframe(meta_data_audio_df)
+            with st.expander("Download Results"):
+                combined_df = pd.concat([df_file_details, meta_data_audio_df])
+                st.dataframe(combined_df)
+                file_download(combined_df)
+
+
+    elif choice=='Document_Files':
+        st.subheader('Document MetaData Extractor')
+        text_file = st.file_uploader("Upload File", type=["PDF"])
+        if text_file is not None:
+            col1, col2 = st.columns([1, 2])
+
+            with col1:
+                with st.expander("File Stats"):
+                    file_details = {
+                        "FileName": text_file.name,
+                        "FileSize": text_file.size,
+                        "FileType": text_file.type,
+                    }
+                    add_file_details(text_file.name,text_file.type,text_file.size,datetime.now())
+
+                    st.write(file_details)
+                    
+                    statinfo = os.stat(text_file.readable())
+
+                    stats_details = {
+                        "Accessed_Time": get_readable_time(statinfo.st_atime),
+                        "Creation_Time": get_readable_time(statinfo.st_ctime),
+                        "Modified_Time": get_readable_time(statinfo.st_mtime),
+                    }
+                    st.write(stats_details)
+
+                    # Combine All Details
+                    file_details_combined = {
+                        "FileName": text_file.name,
+                        "FileSize": text_file.size,
+                        "FileType": text_file.type,
+                        "Accessed_Time": get_readable_time(statinfo.st_atime),
+                        "Creation_Time": get_readable_time(statinfo.st_ctime),
+                        "Modified_Time": get_readable_time(statinfo.st_mtime),
+                    }
+
+                    # Convert to DataFrame
+                    df_file_details = pd.DataFrame(
+                        list(file_details_combined.items()),
+                        columns=["Meta Tags", "Value"],
+                    )
+            with col2:
+                with st.expander("Metadata"):
+                    pdf_file = PdfReader(text_file)
+                    pdf_info = pdf_file.metadata
+                    df_file_details_with_pdf = pd.DataFrame(
+                        list(pdf_info.items()), columns=["Meta Tags", "Value"]
+                    )
+
+                    st.dataframe(df_file_details_with_pdf)
+
+            with st.expander("Download Results"):
+                pdf_combined_df = pd.concat([df_file_details, df_file_details_with_pdf])
+                st.dataframe(pdf_combined_df)
+                file_download(pdf_combined_df)
+
+    elif choice=='Analytics':
+        st.subheader('Analytics')
+        uploaded_files= view_all_data() 
+        df=pd.DataFrame(uploaded_files,columns=['Filename','Filetype','Filesize','UploadDate'])
+        with st.expander('Monitor'):
+            st.success('View all uploaded files')
+            st.dataframe(df)
+
+        #Monitor uploads
+        
+
+
+
+
+
+
+
+

Text_analysis.py

@@ -0,0 +1,179 @@
+import streamlit as st
+import pandas as pd
+import streamlit.components.v1 as stc
+import docx2txt
+
+# NLP Package-used for text analysis
+import nltk
+from nltk.tokenize import word_tokenize
+from nltk.tag import pos_tag
+from nltk.stem import WordNetLemmatizer
+from nltk.corpus import stopwords
+# from nltk import ne_chunk
+from nltk.tag import StanfordNERTagger
+
+from collections import Counter
+
+from textblob import TextBlob
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from wordcloud import WordCloud
+
+import base64
+import time
+from app_utils import *
+
+HTML_BANNER = """
+    <div style="background-color:green;padding:10px;border-radius:10px">
+    <h1 style="color:white;text-align:center;">Text Analysis App </h1>
+    </div>
+    """
+
+def text_analysis():
+    stc.html(HTML_BANNER)
+    menu=['Text-analysis','Upload_Files']
+    
+    choice=st.sidebar.selectbox('Menu',menu)
+    if choice=='Text-analysis':
+        st.subheader('Analyse Text')
+        text=st.text_area("Enter the text to anlayze")
+        if (st.button("Analyze")):
+            st.success("Success")
+            with st.expander('Original Text'):
+                st.write(text)
+            with st.expander('Text Analysis'):
+                token_analysis=nlp_analysis(text)
+                st.dataframe(token_analysis)
+            with st.expander('Entitites'):
+                entity_result=find_entities(text)
+                stc.html(entity_result, height=100, scrolling=True)
+            
+            col1,col2=st.columns(2)
+
+            with col1:
+
+                with st.expander("Word Stats"):
+                    st.info("Word Statistics")
+                    docx = nt.TextFrame(text)
+                    st.write(docx.word_stats())
+
+                with st.expander("Top keywords"):
+                    keywords=get_most_common_tokens(text)
+                    st.write(keywords)
+
+                with st.expander('Tagged Keywords'):
+                    data= pos_tag(text)
+                    st.dataframe(data)
+                    visualize_tags=tag_visualize(data)
+                    stc.html(visualize_tags,scrolling=True)
+
+
+                with st.expander("Sentiment"):
+                    sent_result=get_semantics(text)
+                    st.write(sent_result)
+
+            with col2:
+
+                with st.expander("Plot word freq"):
+                    try:
+                          fig, ax = plt.subplots()
+                          most_common_tokens = dict(token_analysis["Token"].value_counts())
+                          sns.countplot(data=token_analysis[token_analysis["Token"].isin(most_common_tokens)], x="Token", ax=ax)
+                          ax.set_xlabel('PoS')
+                          ax.set_ylabel('Frequency')
+                          ax.tick_params(axis='x' , rotation=45)
+                          st.pyplot(fig)
+                    except:
+                        st.warning('Insufficient data')
+
+                with st.expander("Plot part of speech"):
+                      try:
+                          fig, ax = plt.subplots()
+                          most_common_tokens = dict(token_analysis["Position"].value_counts())
+                          sns.countplot(data=token_analysis[token_analysis["Position"].isin(most_common_tokens)], x="Position", ax=ax)
+                          ax.set_xlabel('PoS')
+                          ax.set_ylabel('Frequency')
+                          ax.tick_params(axis='x' , rotation=45)
+                          st.pyplot(fig)
+                      except:
+                          st.warning('Insufficient data')
+                   
+                with st.expander("Plot word cloud"):
+                    try:
+                        plot_wordcloud(text)
+                    except:
+                        st.warning('Insufficient data')
+
+            
+            with st.expander('Download Results'):
+                file_download(token_analysis)
+
+
+
+
+
+
+    elif choice == 'Upload_Files':
+        text_file = st.file_uploader('Upload Files', type=['docx'])
+        if text_file is not None:
+            if text_file.type == 'text/plain':
+                text = str(text_file.read(), "utf-8")
+            else:
+                text = docx2txt.process(text_file)
+
+            if (st.button("Analyze")):
+                with st.expander('Original Text'):
+                    st.write(text)
+                with st.expander('Text Analysis'):
+                    token_analysis = nlp_analysis(text)
+                    st.dataframe(token_analysis)
+                with st.expander('Entities'):
+                    entity_result = find_entities(text)
+                    stc.html(entity_result, height=100, scrolling=True)
+
+                col1, col2 = st.columns(2)
+
+                with col1:
+                    with st.expander("Word Stats"):
+                        st.info("Word Statistics")
+                        docx = nt.TextFrame(text)
+                        st.write(docx.word_stats())
+
+                    with st.expander("Top keywords"):
+                        keywords = get_most_common_tokens(text)
+                        st.write(keywords)
+
+                    with st.expander("Sentiment"):
+                        sent_result = get_semantics(text)
+                        st.write(sent_result)
+
+                with col2:
+                    with st.expander("Plot word freq"):
+                        fig, ax = plt.subplots()
+                        num_tokens = 10  # Adjust the number of tokens to display as desired
+                        most_common_tokens = dict(token_analysis["Token"].value_counts().head(num_tokens))
+                        sns.countplot(data=token_analysis[token_analysis["Token"].isin(most_common_tokens)], x="Token", ax=ax)
+                        ax.set_xlabel('Token')
+                        ax.set_ylabel('Frequency')
+                        ax.tick_params(axis='x', rotation=45)
+                        st.pyplot(fig)
+
+                    with st.expander("Plot part of speech"):
+                        fig, ax = plt.subplots()
+                        most_common_tokens = dict(token_analysis["Position"].value_counts())
+                        sns.countplot(data=token_analysis[token_analysis["Position"].isin(most_common_tokens)], x="Position", ax=ax)
+                        ax.set_xlabel('PoS')
+                        ax.set_ylabel('Frequency')
+                        ax.tick_params(axis='x', rotation=45)
+                        st.pyplot(fig)
+
+                    with st.expander("Plot word cloud"):
+                        plot_wordcloud(text)
+
+                with st.expander('Download Results'):
+                    file_download(token_analysis)
+
+
+
+

app.py

@@ -0,0 +1,125 @@
+import streamlit as st
+import sumy
+
+# using sumy library for summarization
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.summarizers.text_rank import TextRankSummarizer
+from sumy.nlp.tokenizers import Tokenizer
+import pandas as pd
+import matplotlib.pyplot as plt
+# import seaborn
+from transformers import BartForConditionalGeneration, BartTokenizer
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+from rouge import Rouge
+import altair as at
+import torch
+from Text_analysis import *
+from Metadata import *
+from app_utils import *
+
+
+HTML_BANNER = """
+    <div style="background-color:lightgreen;padding:10px;border-radius:10px">
+    <h1 style="color:white;text-align:center;">Summary app </h1>
+    </div>
+    """
+
+
+def main():
+    menu=['Summarization','Text-Analysis','Meta-Data']
+    choice=st.sidebar.selectbox("Menu",menu)
+
+
+    if choice=='Summarization':
+        stc.html(HTML_BANNER)
+        st.subheader('summarization')
+        raw_text=st.text_area("Enter the text you want to summarize")
+        if st.button("Summarize"):
+            with st.expander("Original Text"):
+                st.write(raw_text)
+            c1, c2 = st.columns(2)
+
+            with c1:
+                with st.expander("LexRank Summary"):
+                    summary = sumy_summarizer(raw_text)
+                    document_len={"Original":len(raw_text),
+                                  "Summary":len(summary)
+                                   }
+                    st.write(document_len)
+                    st.write(summary)
+                    st.info("Rouge Score")
+                    score=evaluate_summary(summary,raw_text)
+                    st.write(score.T)
+                    st.subheader(" ")
+                    score['metrics']=score.index
+                    c=at.Chart(score).mark_bar().encode(
+                        x='metrics',y='rouge-1'
+                    )
+                    st.altair_chart(c)
+
+            with c2:
+                with st.expander("TextRank Summary"):
+                    text_summary=sumy_text_summarizer(raw_text)
+                    document_len={"Original":len(raw_text),
+                                  "Summary":len(summary)
+                                   }
+                    st.write(document_len)
+                    st.write(text_summary)
+
+                    st.info("Rouge Score")
+                    score=evaluate_summary(text_summary,raw_text)
+                    st.write(score.T)
+                    st.subheader(" ")
+                    score['metrics']=score.index
+                    c=at.Chart(score).mark_bar().encode(
+                        x='metrics',y='rouge-1'
+                    )
+                    st.altair_chart(c)
+
+            st.subheader("Bart Sumary")
+            with st.expander("Bart Summary"):
+                bart_summ = bart_summary(raw_text)
+                document_len={"Original":len(raw_text),
+                                  "Summary":len(summary)
+                                   }
+                st.write(document_len)
+                st.write(bart_summ)
+                st.info("Rouge Score")
+                score=evaluate_summary(bart_summ,raw_text)
+                st.write(score.T)
+                st.subheader(" ")
+                score['metrics']=score.index
+                c=at.Chart(score).mark_bar().encode(
+                        x='metrics',y='rouge-1'
+                    )
+                st.altair_chart(c)
+
+            st.subheader("T5 Sumarization")
+            with st.expander("T5 Summary"):
+                T5_sum = T5_summary(raw_text)
+                document_len={"Original":len(raw_text),
+                                  "Summary":len(summary)
+                                   }
+                st.write(document_len)
+                st.write(T5_sum)
+                st.info("Rouge Score")
+                score=evaluate_summary(T5_sum,raw_text)
+                st.write(score.T)
+                st.subheader(" ")
+                score['metrics']=score.index
+                c=at.Chart(score).mark_bar().encode(
+                        x='metrics',y='rouge-1'
+                    )
+                st.altair_chart(c)
+
+
+    elif choice=='Text-Analysis':
+        text_analysis()
+    else:
+        metadata()
+
+
+if __name__=='__main__':
+    main()

app_utils.py

@@ -0,0 +1,202 @@
+import streamlit as st
+import pandas as pd
+import streamlit.components.v1 as stc
+import nltk
+
+# NLP Package-used for text analysis
+from sumy.parsers.plaintext import PlaintextParser
+from nltk.tokenize import word_tokenize
+from nltk.tag import pos_tag
+from nltk.stem import WordNetLemmatizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.summarizers.text_rank import TextRankSummarizer
+from nltk.corpus import stopwords
+from nltk.tokenize import sent_tokenize
+from sumy.nlp.tokenizers import Tokenizer
+from rouge import Rouge
+from transformers import BartForConditionalGeneration, BartTokenizer
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+
+# from nltk import ne_chunk
+from nltk.tag import StanfordNERTagger
+
+from collections import Counter
+
+from textblob import TextBlob
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from wordcloud import WordCloud
+
+import base64
+import time
+stanford_ner_jar = '/Users/ujjwalbansal/Desktop/Summary-app/stanford-ner-2020-11-17/stanford-ner.jar'
+# Path to the pre-trained NER model file
+stanford_ner_model = '/Users/ujjwalbansal/Desktop/Summary-app/stanford-ner-2020-11-17/classifiers/english.all.3class.distsim.crf.ser.gz'
+
+timestr = time.strftime("%Y%m%d-%H%M%S")
+
+
+# from spacy import displacy
+
+
+#Text cleaning packages 
+# removing stopwords, removing special characters, removing URLs, normalizing text, removing HTML tags, correcting common spelling mistakes,
+import neattext as nt
+import neattext.functions as nfx
+
+
+HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid red; border-radius: 0.25rem; padding: 1rem";>{}
+</div> 
+"""
+
+def evaluate_summary(summary,reference):
+    r=Rouge()
+    eval_score=r.get_scores(summary,reference)
+    eval_score_df=pd.DataFrame(eval_score[0])
+    return eval_score_df
+
+
+def bart_summary(docx):
+    model=BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
+    tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
+    inputs = tokenizer.batch_encode_plus([docx], truncation=True, padding='longest', max_length=1024, return_tensors='pt')
+    summary_ids = model.generate(inputs['input_ids'], num_beams=6, max_length=100, early_stopping=True)
+    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+    return  summary
+
+def T5_summary(docx):
+    model = T5ForConditionalGeneration.from_pretrained('t5-base')
+    tokenizer = T5Tokenizer.from_pretrained('t5-base')
+    input_text = "summarize: " + docx
+    input_ids = tokenizer.encode(input_text, return_tensors='pt')
+    summary_ids = model.generate(input_ids, max_length=100, num_beams=4, early_stopping=True)
+    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+    return summary
+    
+def sumy_summarizer(docx,num=5):
+    parser=PlaintextParser.from_string(docx,Tokenizer("english"))
+    lex_summ=LexRankSummarizer()
+    summary=lex_summ(parser.document,sentences_count= num)
+    summary_list=[str(sentence) for sentence in summary]
+    result=' '.join(summary_list)
+    return result
+
+def sumy_text_summarizer(docx, num=5):
+    parser = PlaintextParser.from_string(docx, Tokenizer("english"))
+    text_rank_summarizer = TextRankSummarizer()
+    summary = text_rank_summarizer(parser.document, sentences_count=num)
+    summary_list = [str(sentence) for sentence in summary]
+    result = ' '.join(summary_list)
+    return result
+
+
+def nlp_analysis(text):
+    token_data = []
+    tokens=word_tokenize(text)
+    tagged_tokens = pos_tag(tokens) #categorize into nouns, verbs, adjectives, adverbs, pronouns etc
+    stop_words = set(stopwords.words('english')) #check for words like a", "an", "the", "is", "in"
+    lemmatizer = WordNetLemmatizer() #preprocessing
+    for token in tagged_tokens:
+        token_text=token[0]
+        token_shape = None
+        token_pos = token[1] # "," - Comma CC - Coordinating conjunction DT - Determiner NN - Noun VBD - Past tense verb PRP - Personal pronoun VBD - Past tense verb
+        token_lemma = lemmatizer.lemmatize(token_text)
+        token_is_alpha = token_text.isalpha()
+        token_is_stop = token_text.lower() in stop_words
+        token_data.append([token_text,token_shape,token_pos,token_lemma,token_is_alpha,token_is_stop])
+    df=pd.DataFrame(token_data,columns=['Token','Shape','Position','lemma','Contains_Alphabets','Contains_Stop_words'])
+    return df
+
+
+def find_entities(text):
+    stan = StanfordNERTagger(stanford_ner_model, stanford_ner_jar)
+    text=text.replace("\n\n","\n")
+    tokens = nltk.word_tokenize(text)
+    tagged_tokens = stan.tag(tokens)
+    entities = [(token, tag) for token, tag in tagged_tokens if tag != 'O']
+    entities=HTML_WRAPPER.format(entities)
+    return entities
+
+
+def file_download(data):
+    csv_file= data.to_csv()
+    b64=base64.b64encode(csv_file.encode()).decode()
+    new_filename="result_{}.csv".format(timestr)
+    st.markdown('### 🗃️ Download csv file ')
+    href=f'<a href="data:file/csv;base64,{b64}" download="{new_filename}"> Click Here! </a>'
+    st.markdown(href, unsafe_allow_html=True)
+
+def get_most_common_tokens(text):
+    word_tokens=Counter(text.split())
+    most_common=dict(word_tokens.most_common(len(text)))
+    return most_common
+
+
+def get_semantics(text):
+    blob=TextBlob(text)
+    sentiment=blob.sentiment
+    return sentiment
+
+def plot_wordcloud(text):
+    text_workcloud= WordCloud().generate(text) #size indicates its frequency
+    fig=plt.figure()
+    plt.imshow(text_workcloud,interpolation='bilinear')
+    plt.axis('off')
+    st.pyplot(fig)
+
+def pos_tags(text):
+    blob=TextBlob(text)
+    tagged_text=blob.tags
+    tagged_df=pd.DataFrame(tagged_text,columns=['tokens','tags'])
+    return tagged_df
+
+TAGS = {
+            'NN'   : 'green',
+            'NNS'  : 'green',
+            'NNP'  : 'green',
+            'NNPS' : 'green',
+            'VB'   : 'blue',
+            'VBD'  : 'blue',
+            'VBG'  : 'blue',
+            'VBN'  : 'blue',
+            'VBP'  : 'blue',
+            'VBZ'  : 'blue',
+            'JJ'   : 'red',
+            'JJR'  : 'red',
+            'JJS'  : 'red',
+            'RB'   : 'cyan',
+            'RBR'  : 'cyan',
+            'RBS'  : 'cyan',
+            'IN'   : 'darkwhite',
+            'POS'  : 'darkyellow',
+            'PRP$' : 'magenta',
+            'PRP$' : 'magenta',
+            'DET'   : 'black',
+            'CC'   : 'black',
+            'CD'   : 'black',
+            'WDT'  : 'black',
+            'WP'   : 'black',
+            'WP$'  : 'black',
+            'WRB'  : 'black',
+            'EX'   : 'yellow',
+            'FW'   : 'yellow',
+            'LS'   : 'yellow',
+            'MD'   : 'yellow',
+            'PDT'  : 'yellow',
+            'RP'   : 'yellow',
+            'SYM'  : 'yellow',
+            'TO'   : 'yellow',
+            'None' : 'off'
+        }
+
+def tag_visualize(tagged_df):
+    colored_text=[]
+    for i in tagged_df:
+        if i[1] in TAGS.keys():
+            token=i[0]
+            color_of_text=TAGS.get(i[1])
+            changed_text='<span style=color:{}>{}</span>'.format(color_of_text,token)
+            colored_text.append(changed_text)
+    result=''.join(colored_text)
+    return result