LijinDurairaj commited on
Commit
f93a065
1 Parent(s): 0a85786

resolving error

Browse files
Files changed (2) hide show
  1. app.py +131 -0
  2. requirements.txt +6 -0
app.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from haystack.document_stores import FAISSDocumentStore
2
+ from haystack.nodes import DensePassageRetriever, FARMReader
3
+ from haystack.pipelines import ExtractiveQAPipeline, DocumentSearchPipeline
4
+ from haystack.utils import clean_wiki_text, print_answers
5
+ from sentence_transformers import SentenceTransformer
6
+ from haystack.nodes import EmbeddingRetriever
7
+
8
+ import streamlit as st
9
+ import os
10
+ import pandas as pd
11
+ from datetime import datetime
12
+ import json
13
+
14
+
15
+
16
+ def intializeFAISS(docs):
17
+ try:
18
+ db_file_name='faiss_document_store.db'
19
+ faiss_index_file='faiss_index'
20
+
21
+ if 'reader' not in st.session_state:
22
+ st.session_state.reader=FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=False)
23
+
24
+ if os.path.exists(faiss_index_file):
25
+ document_store=FAISSDocumentStore(
26
+ sql_url=f"sqlite:///{db_file_name}",
27
+ index=faiss_index_file,
28
+ embedding_dim=384
29
+ )
30
+ else:
31
+ document_store = FAISSDocumentStore(embedding_dim=384, faiss_index_factory_str="Flat")
32
+
33
+ if 'retriever' not in st.session_state:
34
+ st.session_state.retriever = EmbeddingRetriever(
35
+ document_store=document_store,
36
+ embedding_model="sentence-transformers/all-MiniLM-L6-v2",
37
+ model_format="sentence_transformers",
38
+ use_gpu=False
39
+ )
40
+
41
+ document_store.delete_all_documents()
42
+ document_store.write_documents(docs)
43
+ document_store.update_embeddings(st.session_state.retriever)
44
+ document_store.save(index_path=faiss_index_file)
45
+
46
+ except Exception as ex:
47
+ print('--error--')
48
+ print('--intialization method--')
49
+ print(ex)
50
+
51
+
52
+ def prepare_doc(r):
53
+ _content=f'''{r['Salutation']} {r['Initial']} {r['Name']}, Employee Id is {r['Employee Id']} and user id is {r['User Id']}, is from country {r['Country']}, Supervisor / Manager name is {r['Supervisor / Manager']} and Contracting Company is {r['Contracting Company']}, Primary Industry is {r['Primary Industry']} and Secondary Industry is {r['Secondary Industry']}, there Sector is {r['Sector']}, they are expertise in {r['Expertise']}, there role is {r['Industry Role']}. There last Last Promotion Date is {r['Last Promotion Date']} and {r['Last Promotional Level']}, There Job Title is {r['Job Title']}, they are working here since {r['Professional Since']}, there hired date is {r['Hired Date']}, there Relevant is {r['Relevant']}, Employee Sponser is {r['Employee Sponser']}, Job Description is {r['Job Description']}, Emergency Contact Name is {r['Emergency Contact Name']} and Emergency Contact Number is {r['Emergency Contact Number']}, Regional Supervisor is {r['Regional Supervisor']} Office Supervisor is {r['Office Supervisor']} Engagement Supervisor is {r['Engagement Supervisor']} '''
54
+ return {
55
+ 'content':_content,
56
+ 'meta':{
57
+ "Salutation":r["Salutation"],
58
+ "Initial":r["Initial"],
59
+ "User Id" :r["User Id"],
60
+ "Name" :r["Name"],
61
+ "Employee Id" :r["Employee Id"],
62
+ "Country":r["Country"],
63
+ "Supervisor / Manager" :r["Supervisor / Manager"],
64
+ "Contracting Company" :r["Contracting Company"],
65
+ "Primary Industry":r["Primary Industry"],
66
+ "Secondary Industry" :r["Secondary Industry"],
67
+ "Sector" :r["Sector"],
68
+ "Expertise":r["Expertise"],
69
+ "Industry Role":r["Industry Role"],
70
+ "Designation" :r["Designation"],
71
+ "Grade" :r["Grade"],
72
+ "Target Chargeability %" :r["Target Chargeability %"],
73
+ "Charge Out Rate":r["Charge Out Rate"],
74
+ "Last Promotion Date" :r["Last Promotion Date"],
75
+ "Last Promotional Level" :r["Last Promotional Level"],
76
+ "Job Title":r["Job Title"],
77
+ "Professional Since" :r["Professional Since"],
78
+ "Hired Date" :r["Hired Date"],
79
+ "Relevant":r["Relevant"],
80
+ "Employee Sponser":r["Employee Sponser"],
81
+ "Job Description" :r["Job Description"],
82
+ "Emergency Contact Name" :r["Emergency Contact Name"],
83
+ "Emergency Contact Number":r["Emergency Contact Number"],
84
+ "Regional Supervisor" :r["Regional Supervisor"],
85
+ "Office Supervisor" :r["Office Supervisor"],
86
+ "Engagement Supervisor":r["Engagement Supervisor"]
87
+ }
88
+ }
89
+
90
+
91
+ def on_submission(question):
92
+ try:
93
+ print('reader')
94
+ print(st.session_state.reader)
95
+ print('retriever')
96
+ print(st.session_state.retriever)
97
+ qa_pipeline=ExtractiveQAPipeline(reader=st.session_state.reader,retriever=st.session_state.retriever)
98
+ prediction=qa_pipeline.run(query=question,params={
99
+ "Retriever":{"top_k":5},
100
+ "Reader":{"top_k":1}
101
+ })
102
+
103
+ if prediction:
104
+ df=st.dataframe(prediction['answers'][0])
105
+ print(type(df))
106
+
107
+
108
+ except Exception as ex:
109
+ print('--error--')
110
+ print('--on_submission method--')
111
+ print(ex)
112
+
113
+ uploaded_file=st.file_uploader(label='please upload your file',type=['csv'])
114
+
115
+ if uploaded_file is not None:
116
+ df=pd.read_csv(uploaded_file)
117
+ df['formatted_content']=df.apply(prepare_doc,axis=1)
118
+ document=df['formatted_content'].to_numpy()
119
+ intializeFAISS(docs=document)
120
+
121
+ #intialize form
122
+ with st.form(key='workforce_management'):
123
+ st.title('workforce management')
124
+
125
+ question=st.text_input(label='please ask your question.')
126
+ submitted=st.form_submit_button(label='submit')
127
+
128
+ if submitted and question:
129
+ on_submission(question)
130
+
131
+
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ sentence_transformers
2
+ transformers
3
+ SQLAlchemy==1.4.46
4
+ farm-haystack==1.26.3 #not installed
5
+ faiss-cpu
6
+ streamlit