adding textrank

appStore/sdg_analysis.py

@@ -14,7 +14,7 @@ from docx.shared import Pt
 from docx.enum.style import WD_STYLE_TYPE
 from utils.sdg_classifier import sdg_classification
 from utils.sdg_classifier import runSDGPreprocessingPipeline
-from utils.keyword_extraction import keywordExtraction
+from utils.keyword_extraction import keywordExtraction, textrank
 import logging
 logger = logging.getLogger(__name__)
 
@@ -59,9 +59,13 @@ def app():
                     keywordList = []
                     for label in sdg_labels:
                         sdgdata = " ".join(df[df.SDG == label].text.to_list())
-                        list_ = keywordExtraction(label,[sdgdata])
-                        keywordList.append({'SDG':label, 'Keywords':list_})
+                        tfidflist_ = keywordExtraction(label,[sdgdata])
+                        textranklist_ = textrank(sdgdata, words = 20)
+                        keywordList.append({'SDG':label, 'TFIDF Keywords':tfidflist_, 'TEXT RANK':textranklist_})
                     keywordsDf = pd.DataFrame(keywordList)
+
+
+
                     
 
 
@@ -83,11 +87,13 @@ def app():
                     with c5:
                         st.pyplot(fig)
                     
-                    st.markdown("##### What keywords are present under SDG labels? #####")
+                    st.markdown("##### What keywords are present under SDG classified text? #####")
+                    st.write("TFIDF BASED")
 
-                    c1, c2, c3 = st.columns([1, 3, 1])
+                    c1, c2, c3 = st.columns([1, 10, 1])
                     with c2:
                         st.table(keywordsDf)
+                    
                         
                     c7, c8, c9 = st.columns([1, 10, 1])
                     with c8:

paramconfig.cfg

@@ -30,4 +30,5 @@ SPLIT_OVERLAP_WORD = 10
 SPLIT_OVERLAP_SENTENCE = 1
 
 [tfidf]
-TOP_N = 10
+TOP_N = 20
+TEXTRANK_WORDS = 20

requirements.txt

@@ -13,5 +13,6 @@ seaborn==0.11.2
 transformers==4.21.2
 st-annotated-text==3.0.0
 markdown==3.4.1
+summa==1.2.0
 python-docx
 streamlit_option_menu

utils/keyword_extraction.py

@@ -7,6 +7,8 @@ import pickle
 from typing import List, Text
 import configparser
 import logging
+from summa import keywords
+
 try:
     from termcolor import colored
 except:
@@ -62,8 +64,12 @@ def keywordExtraction(sdg:int,sdgdata:List[Text]):
     keywords = [keyword for keyword in results]
     return keywords
 
+def textrank(textdata, ratio = 0.1, words = 0):
+    if words == 0:
+        results = keywords.keywords(textdata, ratio= ratio).split("\n")
+    else:
+        results = keywords.keywords(textdata, words= words).split("\n")
 
-
-    
+    return results