ifidf models

docStore/sdg1/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg1/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg10/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg10/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg11/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg11/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg12/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg12/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg13/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg13/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg14/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg14/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg15/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg15/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg2/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg2/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg3/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg3/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg4/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg4/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg5/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg5/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg6/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg6/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg7/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg7/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg8/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg8/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg9/tfidfmodel.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

docStore/sdg9/vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a6c9867a76dec849638864fa1463033b0620f9611dc6cdbac22c1d7a8cbf9b8
+size 2520

utils/keyword_extraction.py

@@ -0,0 +1,42 @@
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+import nltk
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+import pickle
+
+
+def sort_coo(coo_matrix):
+    tuples = zip(coo_matrix.col, coo_matrix.data)
+    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)
+
+def extract_topn_from_vector(feature_names, sorted_items, topn=10):
+    """get the feature names and tf-idf score of top n items"""
+    
+    #use only topn items from vector
+    sorted_items = sorted_items[:topn]
+    score_vals = []
+    feature_vals = []
+    
+    # word index and corresponding tf-idf score
+    for idx, score in sorted_items:
+        
+        #keep track of feature name and its corresponding score
+        score_vals.append(round(score, 3))
+        feature_vals.append(feature_names[idx])
+    #create a tuples of feature,score
+    #results = zip(feature_vals,score_vals)
+    results= {}
+    for idx in range(len(feature_vals)):
+        results[feature_vals[idx]]=score_vals[idx]
+    
+    return results
+
+def keyword_extraction(sdg:int,sdgdata):
+    model_path = "docStore/sdg{}/".format(sdg)
+    vectorizer = pickle.load(open(model_path+'vectorizer.pkl', 'rb'))
+    tfidfmodel = pickle.load(open(model_path+'tfidfmodel.pkl', 'rb'))
+    features = vectorizer.get_feature_names_out()
+    
+
+

utils/preprocessing.py

@@ -79,7 +79,7 @@ class FileConverter(BaseComponent):
             if file_name.endswith('.txt'):
                 converter = TextConverter(remove_numeric_tables=True)
             if file_name.endswith('.docx'):
-                converter = DocxToTextConverter(remove_numeric_tables=True) 
+                converter = DocxToTextConverter() 
         except Exception as e:
             logging.error(e)
             return 

utils/semantic_search.py

@@ -188,6 +188,66 @@ def semanticSearchPipeline(documents:List[Document]):
     return semanticsearch_pipeline, document_store
 
 
+def semanticsearchAnnotator(matches: List[List[int]], document):
+    """
+    Annotates the text in the document defined by list of [start index, end index]
+    Example: "How are you today", if document type is text, matches = [[0,3]]
+    will give answer = "How", however in case we used the spacy matcher then the
+    matches = [[0,3]] will give answer = "How are you". However if spacy is used
+    to find "How" then the matches = [[0,1]] for the string defined above.
+
+    """
+    start = 0
+    annotated_text = ""
+    for match in matches:
+        start_idx = match[0]
+        end_idx = match[1]
+        if check_streamlit():
+            annotated_text = (annotated_text + document[start:start_idx]
+                            + str(annotation(body=document[start_idx:end_idx],
+                            label="Context", background="#964448", color='#ffffff')))
+        else:
+            annotated_text = (annotated_text + document[start:start_idx]
+                            + colored(document[start_idx:end_idx],
+                          "green", attrs = ['bold']))
+        start = end_idx
+    
+    annotated_text = annotated_text + document[end_idx:]
+
+    if check_streamlit():
+
+        st.write(
+                markdown(annotated_text),
+                unsafe_allow_html=True,
+            )
+    else:
+        print(annotated_text)
+    
+
+def semantic_search(query:Text,documents:List[Document]):
+    """
+    Performs the Semantic search on the List of haystack documents which is 
+    returned by preprocessing Pipeline.
+
+    Params
+    -------
+    query: Keywords that need to be searche in documents.
+    documents: List fo Haystack documents returned by preprocessing pipeline.
+    
+    """
+    semanticsearch_pipeline, doc_store = semanticSearchPipeline(documents)
+    results = semanticsearch_pipeline.run(query = query)
+    st.markdown("##### Top few semantic search results #####")
+    for i,answer in enumerate(results['answers']):
+        temp = answer.to_dict()
+        start_idx = temp['offsets_in_document'][0]['start']
+        end_idx = temp['offsets_in_document'][0]['end']
+        match = [[start_idx,end_idx]]
+        doc = doc_store.get_document_by_id(temp['document_id']).content
+        st.write("Result {}".format(i+1))
+        semanticsearchAnnotator(match, doc)
+
+
 
     # if 'document_store' in st.session_state:
     #     document_store = st.session_state['document_store']
@@ -264,63 +324,4 @@ def semanticSearchPipeline(documents:List[Document]):
     # semanticsearch_pipeline.add_node(component = reader, name = "FARMReader",
     #                                 inputs= ["EmbeddingRetriever"])
     
-    # return semanticsearch_pipeline, document_store
-
-def semanticsearchAnnotator(matches: List[List[int]], document):
-    """
-    Annotates the text in the document defined by list of [start index, end index]
-    Example: "How are you today", if document type is text, matches = [[0,3]]
-    will give answer = "How", however in case we used the spacy matcher then the
-    matches = [[0,3]] will give answer = "How are you". However if spacy is used
-    to find "How" then the matches = [[0,1]] for the string defined above.
-
-    """
-    start = 0
-    annotated_text = ""
-    for match in matches:
-        start_idx = match[0]
-        end_idx = match[1]
-        if check_streamlit():
-            annotated_text = (annotated_text + document[start:start_idx]
-                            + str(annotation(body=document[start_idx:end_idx],
-                            label="ANSWER", background="#964448", color='#ffffff')))
-        else:
-            annotated_text = (annotated_text + document[start:start_idx]
-                            + colored(document[start_idx:end_idx],
-                          "green", attrs = ['bold']))
-        start = end_idx
-    
-    annotated_text = annotated_text + document[end_idx:]
-
-    if check_streamlit():
-
-        st.write(
-                markdown(annotated_text),
-                unsafe_allow_html=True,
-            )
-    else:
-        print(annotated_text)
-    
-
-def semantic_search(query:Text,documents:List[Document]):
-    """
-    Performs the Semantic search on the List of haystack documents which is 
-    returned by preprocessing Pipeline.
-
-    Params
-    -------
-    query: Keywords that need to be searche in documents.
-    documents: List fo Haystack documents returned by preprocessing pipeline.
-    
-    """
-    semanticsearch_pipeline, doc_store = semanticSearchPipeline(documents)
-    results = semanticsearch_pipeline.run(query = query)
-    st.markdown("##### Top few semantic search results #####")
-    for i,answer in enumerate(results['answers']):
-        temp = answer.to_dict()
-        start_idx = temp['offsets_in_document'][0]['start']
-        end_idx = temp['offsets_in_document'][0]['end']
-        match = [[start_idx,end_idx]]
-        doc = doc_store.get_document_by_id(temp['document_id']).content
-        st.write("Result {}".format(i+1))
-        semanticsearchAnnotator(match, doc)
+    # return semanticsearch_pipeline, document_store