Upload 7 files

app.py

@@ -0,0 +1,14 @@
+import gradio as gr
+from load_data import *
+
+def chitti(query):
+    re_table = search(query)
+    return [re_table[0][0],re_table[0][3],re_table[1][0],re_table[1][3]]
+
+demo = gr.Interface(
+    fn=chitti,
+    inputs=["text"],
+    outputs=["text","text","text","text"],
+)
+demo.launch(share=True)
+

corpus.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8388b97d5d8045bc3a9fc29f6793001123368ed44de62ff9a26d5f33f4a3ff56
+size 5803755

data_url.csv

@@ -0,0 +1,56 @@
+file_name,link
+0,ccservices_new.txt,https://www.cc.iitb.ac.in/
+1,file1.txt,https://drive.google.com/file/d/1NaYLmXUc1DzhxZ3ps9fBp9ZN5i1qrtZy/view?usp=sharing
+2,file10.txt,https://drive.google.com/file/d/1sIvXZFTs051Si7CC37b35gnpN4BRuKmf/view?usp=sharing
+3,file11.txt,https://drive.google.com/file/d/1dbQcZsco3mRjBZNz8BDdXurzbKuWmbTE/view?usp=sharing
+4,file12.txt,https://drive.google.com/file/d/17ky3Ayl29Q1mWAYqivV-5MXVJ4FUVkbK/view?usp=sharing
+5,file13.txt,https://drive.google.com/file/d/1bZ1ZORsXS3bsDeQupEwdkdicbcfdsuzZ/view?usp=sharing
+6,file14.txt,https://drive.google.com/file/d/1eTICcbswfGC3yFl4hcajy6JahRjwOFTD/view?usp=sharing
+7,file15.txt,https://drive.google.com/file/d/1JQo8NkjtdWtrAXScSMC6Bp0xG6-EAbCo/view?usp=sharing
+8,file17.txt,https://drive.google.com/file/d/1-r5uvlzFcNsChsXzqvpLjTVQ3FWhDZub/view?usp=sharing
+9,file18.txt,https://drive.google.com/file/d/1eV3DfnbSspxT_GJ755iemXSuacqK0OMK/view?usp=sharing
+10,file2.txt,https://drive.google.com/file/d/1zn6hPZe1fJ3otvWvXEyq2V991sTfYo1v/view?usp=sharing
+11,file20_new.txt,https://drive.google.com/file/d/11rJhoC34ONDXPo7r9U6ynlLFIp_CFqjG/view?usp=sharing
+12,file21_new.txt,https://drive.google.com/file/d/1Rc91IlWYC0l3u0GMlFN8u6P1Lqg3bp43/view?usp=sharing
+13,file22.txt,https://drive.google.com/file/d/155w3J-KcmnTtpRIZEMqU8Sy2w_85nTlA/view?usp=sharing
+14,file23.txt,https://drive.google.com/file/d/1h1TCBnl7__c63F9IQR0P8wvsIUGxIbU4/view?usp=sharing
+15,file24.txt,https://drive.google.com/file/d/1FKa2uXENxCeUPm0j51uYVmUQfnUOtAvU/view?usp=sharing
+16,file3.txt,https://drive.google.com/file/d/1yMO7AclQWfvuIgRvP-ACEAIpwIayIRrV/view?usp=sharing
+17,file4.txt,https://drive.google.com/file/d/1Dz7vH8vZggfezr5sgpYI1ZsV41VOk23o/view?usp=sharing
+18,file5.txt,https://drive.google.com/file/d/1BCpQwWA5RL1adaZkJnv7U7-qWGLoPi4d/view?usp=sharing
+19,file6.txt,https://drive.google.com/file/d/1pP5kK9r350pmYIWJl6aRdMEe06XMf3ny/view?usp=sharing
+20,file7.txt,https://drive.google.com/file/d/1LBFWcznes0Xyth9HzFepJezBtl2O0ItJ/view?usp=sharing
+21,file8.txt,https://drive.google.com/file/d/1F56u2ro-qmCnwjV8jsjQOOLYY4aOu0Ed/view?usp=sharing
+22,file9.txt,https://drive.google.com/file/d/19pQBbTLy-7OI4hKwoaQ9pehsVKpLWCmH/view?usp=sharing
+23,getting_started_new.txt,https://www.cc.iitb.ac.in/
+24,HallmanagerDuties.txt,https://gymkhana.iitb.ac.in/hostels/#/hall-manager-duties
+25,howto_new.txt,https://www.cc.iitb.ac.in/
+26,ismp0.txt,https://smp.gymkhana.iitb.ac.in/incoming_introduction.php
+27,ismp1.txt,https://smp.gymkhana.iitb.ac.in/incoming_things_to_do.php
+28,ismp10.txt,https://smp.gymkhana.iitb.ac.in/academics_cpi.php
+29,ismp11.txt,https://smp.gymkhana.iitb.ac.in/academics_bc.php
+30,ismp12.txt,https://smp.gymkhana.iitb.ac.in/academics_idddp.php
+31,ismp13.txt,https://smp.gymkhana.iitb.ac.in/academics_semex.php
+32,ismp14.txt,https://smp.gymkhana.iitb.ac.in/life_intro.php
+33,ismp15.txt,https://smp.gymkhana.iitb.ac.in/life_intro.php
+34,ismp16.txt,https://smp.gymkhana.iitb.ac.in/life_campus.php
+35,ismp17.txt,https://smp.gymkhana.iitb.ac.in/life_support.php
+36,ismp18.txt,https://smp.gymkhana.iitb.ac.in/life_culture.php
+37,ismp19.txt,https://smp.gymkhana.iitb.ac.in/extra_curricular.php
+38,ismp2.txt,https://smp.gymkhana.iitb.ac.in/incoming_accomadation.php
+39,ismp20.txt,https://smp.gymkhana.iitb.ac.in/extra_gym_sports.php#gymkhana
+40,ismp21.txt,https://smp.gymkhana.iitb.ac.in/extra_culture.php
+41,ismp22.txt,https://smp.gymkhana.iitb.ac.in/extra_media.php
+42,ismp23.txt,https://smp.gymkhana.iitb.ac.in/extra_gym_sports.php#sports
+43,ismp24.txt,https://smp.gymkhana.iitb.ac.in/extra_technical.php
+44,ismp25.txt,https://smp.gymkhana.iitb.ac.in/extra_ibs.php
+45,ismp3.txt,https://smp.gymkhana.iitb.ac.in/incoming_scholarships.php
+46,ismp4.txt,https://smp.gymkhana.iitb.ac.in/incoming_fee_structure.php
+47,ismp5.txt,https://smp.gymkhana.iitb.ac.in/incoming_letter_to_parents.php
+48,ismp6.txt,https://smp.gymkhana.iitb.ac.in/academics.php
+49,ismp7.txt,https://smp.gymkhana.iitb.ac.in/academics_intro.php
+50,ismp8.txt,https://smp.gymkhana.iitb.ac.in/academics.php#curriculum
+51,ismp9.txt,https://smp.gymkhana.iitb.ac.in/academics.php#departments
+52,network_new.txt,https://www.cc.iitb.ac.in/
+53,policies_new.txt,https://www.cc.iitb.ac.in/
+54,SWC.txt,https://www.iitb.ac.in/swc/en/about-student-wellness-centre

load_data.py

@@ -0,0 +1,138 @@
+import pandas as pd
+import os
+import json
+import re
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+import torch
+import time
+import textwrap
+
+model_bi_encoder = "msmarco-distilbert-base-tas-b"
+model_cross_encoder = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+
+bi_encoder = SentenceTransformer(model_bi_encoder)
+bi_encoder.max_seq_length = 512
+cross_encoder = CrossEncoder(model_cross_encoder)
+
+def collect_data(data_lis,meta_count):
+    new_files = data_lis['file_name'][meta_count:]
+    new_links = data_lis['link'][meta_count:]
+    return new_files,new_links
+
+def merge_text(text_list):
+    i = 0;j = 1
+    k = len(text_list)
+    while j < k:
+        if len(text_list[i].split()) <= 30:
+            text_list[j] = text_list[i] + " " + text_list[j]
+            text_list[i] = " "
+        i += 1;j += 1
+    return [accepted for accepted in text_list if accepted != " "]
+
+def make_data(new_files,new_links,local_path):
+    text = [];links = []
+    for doc in range(len(new_files)):
+        sub_text = [];sub_link = []
+        with open(os.path.join(local_path, new_files[doc]), encoding='utf-8') as f:
+            for line in f.readlines():
+                temp_text = re.sub("\\n", "", line)
+                if temp_text != "":
+                    sub_text.append(temp_text)
+            sub_text = merge_text(sub_text)
+            sub_link = [new_links[doc] for i in range(len(sub_text))]
+            text.extend(sub_text)
+            links.extend(sub_link)
+    return text,links
+
+def get_final_data():
+
+    #Define all the paths
+    meta_path = "meta_data.json"
+    data_lis_path = "data_url.csv"
+    local_path = "Data_final"
+    data_path = "Responses.csv"
+    corpus_path = "corpus.pt"
+
+    # Load the list of data files
+    data_lis = pd.read_csv(data_lis_path)
+
+    # Load the responses.csv file
+    if not(os.path.exists(data_path)):
+        fresh_text = []
+        fresh_link = []
+        fresh_data = {
+            "text": fresh_text,
+            "links": fresh_link
+        }
+        fresh_data = pd.DataFrame(fresh_data)
+        fresh_data.to_csv(data_path)
+    data = pd.read_csv(data_path)
+
+    # Check for any new files; If present add those to responses.csv file
+    # Make changes to corpus.pt accordingly
+    act_count = len(data_lis['file_name'])
+    with open(meta_path, "r") as jsonFile:
+        meta_data = json.load(jsonFile)
+    meta_count = meta_data["data"]["count"]
+
+    if meta_count!=act_count:
+        meta_data["data"]["count"] = act_count
+        with open(meta_path, "w") as jsonFile:
+            json.dump(meta_data, jsonFile)
+        new_files,new_links = collect_data(data_lis,meta_count)
+        text,links = make_data(new_files,new_links,local_path)
+        df = {
+            "text": text,
+            "links":links
+        }
+        df = pd.DataFrame(df)
+        data = pd.concat([data,df])
+        data.to_csv("Responses.csv")
+        if not(os.path.exists(corpus_path)):
+            corpus_embeddings = bi_encoder.encode(data["text"], convert_to_tensor=True, show_progress_bar=True)
+            torch.save(corpus_embeddings, corpus_path)
+        else:
+            corpus_embeddings = torch.load(corpus_path)
+            new_embeddings = bi_encoder.encode(df["text"], convert_to_tensor=True, show_progress_bar=True)
+            corpus_embeddings = torch.cat((corpus_embeddings,new_embeddings),0)
+            torch.save(corpus_embeddings, corpus_path)
+
+    corpus_embeddings = torch.load(corpus_path)
+    return corpus_embeddings,data
+
+
+def search(query):
+    corpus_embeddings,data = get_final_data()
+    question_embedding = bi_encoder.encode(query, convert_to_tensor=True)
+    top_k = 20
+    #be = time.process_time()
+    hits = util.semantic_search(question_embedding, corpus_embeddings, top_k=top_k)
+    #print("Time taken by Bi-encoder:" + str(time.process_time() - be))
+
+    hits = hits[0]
+    cross_inp = [[query, data['text'][hit['corpus_id']]] for hit in hits]
+
+    #ce = time.process_time()
+    cross_scores = cross_encoder.predict(cross_inp)
+    #print("Time taken by Cross-encoder:" + str(time.process_time() - ce))
+
+    # Sort results by the cross-encoder scores
+    for idx in range(len(cross_scores)):
+        hits[idx]['cross-score'] = cross_scores[idx]
+
+
+    hits = sorted(hits, key=lambda x: x['cross-score'], reverse=True)
+    result_table = list()
+    for hit in hits[0:5]:
+        ans = "{}".format(data['text'][hit['corpus_id']].replace("\n", " "))
+        #print(ans)
+        cs = "{}".format(hit['cross-score'])
+        #print(cs)
+        sc = "{}".format(hit['score'])
+        #print(sc)
+        corr_link = "{}".format(data['links'][hit['corpus_id']])
+        wrapper = textwrap.TextWrapper(width=50)
+        ans = wrapper.fill(text=ans)
+        result_table.append([ans,str(cs),str(sc),str(corr_link)])
+        
+    return result_table

meta_data.json

@@ -0,0 +1 @@
+{"data": {"count": 55}}

requirements.txt

@@ -0,0 +1,2 @@
+sentence-transformers
+pandas