Spaces:
Sleeping
Sleeping
LijinDurairaj
commited on
Commit
•
f93a065
1
Parent(s):
0a85786
resolving error
Browse files- app.py +131 -0
- requirements.txt +6 -0
app.py
ADDED
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from haystack.document_stores import FAISSDocumentStore
|
2 |
+
from haystack.nodes import DensePassageRetriever, FARMReader
|
3 |
+
from haystack.pipelines import ExtractiveQAPipeline, DocumentSearchPipeline
|
4 |
+
from haystack.utils import clean_wiki_text, print_answers
|
5 |
+
from sentence_transformers import SentenceTransformer
|
6 |
+
from haystack.nodes import EmbeddingRetriever
|
7 |
+
|
8 |
+
import streamlit as st
|
9 |
+
import os
|
10 |
+
import pandas as pd
|
11 |
+
from datetime import datetime
|
12 |
+
import json
|
13 |
+
|
14 |
+
|
15 |
+
|
16 |
+
def intializeFAISS(docs):
|
17 |
+
try:
|
18 |
+
db_file_name='faiss_document_store.db'
|
19 |
+
faiss_index_file='faiss_index'
|
20 |
+
|
21 |
+
if 'reader' not in st.session_state:
|
22 |
+
st.session_state.reader=FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False)
|
23 |
+
|
24 |
+
if os.path.exists(faiss_index_file):
|
25 |
+
document_store=FAISSDocumentStore(
|
26 |
+
sql_url=f"sqlite:///{db_file_name}",
|
27 |
+
index=faiss_index_file,
|
28 |
+
embedding_dim=384
|
29 |
+
)
|
30 |
+
else:
|
31 |
+
document_store = FAISSDocumentStore(embedding_dim=384, faiss_index_factory_str="Flat")
|
32 |
+
|
33 |
+
if 'retriever' not in st.session_state:
|
34 |
+
st.session_state.retriever = EmbeddingRetriever(
|
35 |
+
document_store=document_store,
|
36 |
+
embedding_model="sentence-transformers/all-MiniLM-L6-v2",
|
37 |
+
model_format="sentence_transformers",
|
38 |
+
use_gpu=False
|
39 |
+
)
|
40 |
+
|
41 |
+
document_store.delete_all_documents()
|
42 |
+
document_store.write_documents(docs)
|
43 |
+
document_store.update_embeddings(st.session_state.retriever)
|
44 |
+
document_store.save(index_path=faiss_index_file)
|
45 |
+
|
46 |
+
except Exception as ex:
|
47 |
+
print('--error--')
|
48 |
+
print('--intialization method--')
|
49 |
+
print(ex)
|
50 |
+
|
51 |
+
|
52 |
+
def prepare_doc(r):
|
53 |
+
_content=f'''{r['Salutation']} {r['Initial']} {r['Name']}, Employee Id is {r['Employee Id']} and user id is {r['User Id']}, is from country {r['Country']}, Supervisor / Manager name is {r['Supervisor / Manager']} and Contracting Company is {r['Contracting Company']}, Primary Industry is {r['Primary Industry']} and Secondary Industry is {r['Secondary Industry']}, there Sector is {r['Sector']}, they are expertise in {r['Expertise']}, there role is {r['Industry Role']}. There last Last Promotion Date is {r['Last Promotion Date']} and {r['Last Promotional Level']}, There Job Title is {r['Job Title']}, they are working here since {r['Professional Since']}, there hired date is {r['Hired Date']}, there Relevant is {r['Relevant']}, Employee Sponser is {r['Employee Sponser']}, Job Description is {r['Job Description']}, Emergency Contact Name is {r['Emergency Contact Name']} and Emergency Contact Number is {r['Emergency Contact Number']}, Regional Supervisor is {r['Regional Supervisor']} Office Supervisor is {r['Office Supervisor']} Engagement Supervisor is {r['Engagement Supervisor']} '''
|
54 |
+
return {
|
55 |
+
'content':_content,
|
56 |
+
'meta':{
|
57 |
+
"Salutation":r["Salutation"],
|
58 |
+
"Initial":r["Initial"],
|
59 |
+
"User Id" :r["User Id"],
|
60 |
+
"Name" :r["Name"],
|
61 |
+
"Employee Id" :r["Employee Id"],
|
62 |
+
"Country":r["Country"],
|
63 |
+
"Supervisor / Manager" :r["Supervisor / Manager"],
|
64 |
+
"Contracting Company" :r["Contracting Company"],
|
65 |
+
"Primary Industry":r["Primary Industry"],
|
66 |
+
"Secondary Industry" :r["Secondary Industry"],
|
67 |
+
"Sector" :r["Sector"],
|
68 |
+
"Expertise":r["Expertise"],
|
69 |
+
"Industry Role":r["Industry Role"],
|
70 |
+
"Designation" :r["Designation"],
|
71 |
+
"Grade" :r["Grade"],
|
72 |
+
"Target Chargeability %" :r["Target Chargeability %"],
|
73 |
+
"Charge Out Rate":r["Charge Out Rate"],
|
74 |
+
"Last Promotion Date" :r["Last Promotion Date"],
|
75 |
+
"Last Promotional Level" :r["Last Promotional Level"],
|
76 |
+
"Job Title":r["Job Title"],
|
77 |
+
"Professional Since" :r["Professional Since"],
|
78 |
+
"Hired Date" :r["Hired Date"],
|
79 |
+
"Relevant":r["Relevant"],
|
80 |
+
"Employee Sponser":r["Employee Sponser"],
|
81 |
+
"Job Description" :r["Job Description"],
|
82 |
+
"Emergency Contact Name" :r["Emergency Contact Name"],
|
83 |
+
"Emergency Contact Number":r["Emergency Contact Number"],
|
84 |
+
"Regional Supervisor" :r["Regional Supervisor"],
|
85 |
+
"Office Supervisor" :r["Office Supervisor"],
|
86 |
+
"Engagement Supervisor":r["Engagement Supervisor"]
|
87 |
+
}
|
88 |
+
}
|
89 |
+
|
90 |
+
|
91 |
+
def on_submission(question):
|
92 |
+
try:
|
93 |
+
print('reader')
|
94 |
+
print(st.session_state.reader)
|
95 |
+
print('retriever')
|
96 |
+
print(st.session_state.retriever)
|
97 |
+
qa_pipeline=ExtractiveQAPipeline(reader=st.session_state.reader,retriever=st.session_state.retriever)
|
98 |
+
prediction=qa_pipeline.run(query=question,params={
|
99 |
+
"Retriever":{"top_k":5},
|
100 |
+
"Reader":{"top_k":1}
|
101 |
+
})
|
102 |
+
|
103 |
+
if prediction:
|
104 |
+
df=st.dataframe(prediction['answers'][0])
|
105 |
+
print(type(df))
|
106 |
+
|
107 |
+
|
108 |
+
except Exception as ex:
|
109 |
+
print('--error--')
|
110 |
+
print('--on_submission method--')
|
111 |
+
print(ex)
|
112 |
+
|
113 |
+
uploaded_file=st.file_uploader(label='please upload your file',type=['csv'])
|
114 |
+
|
115 |
+
if uploaded_file is not None:
|
116 |
+
df=pd.read_csv(uploaded_file)
|
117 |
+
df['formatted_content']=df.apply(prepare_doc,axis=1)
|
118 |
+
document=df['formatted_content'].to_numpy()
|
119 |
+
intializeFAISS(docs=document)
|
120 |
+
|
121 |
+
#intialize form
|
122 |
+
with st.form(key='workforce_management'):
|
123 |
+
st.title('workforce management')
|
124 |
+
|
125 |
+
question=st.text_input(label='please ask your question.')
|
126 |
+
submitted=st.form_submit_button(label='submit')
|
127 |
+
|
128 |
+
if submitted and question:
|
129 |
+
on_submission(question)
|
130 |
+
|
131 |
+
|
requirements.txt
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
sentence_transformers
|
2 |
+
transformers
|
3 |
+
SQLAlchemy==1.4.46
|
4 |
+
farm-haystack==1.26.3 #not installed
|
5 |
+
faiss-cpu
|
6 |
+
streamlit
|