karshreya98
commited on
Commit
β’
d348ade
1
Parent(s):
e48d908
added chnages for open_ai key to avoid merge conflicts
Browse files- .gitignore +3 -0
- app.py +31 -15
- utils/config.py +0 -2
- utils/haystack.py +7 -5
.gitignore
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
.env
|
2 |
+
.vscode
|
3 |
+
*.pyc
|
app.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1 |
-
|
2 |
import streamlit as st
|
3 |
import logging
|
4 |
import os
|
@@ -10,10 +10,10 @@ from utils.config import parser
|
|
10 |
from utils.haystack import start_document_store, query, initialize_pipeline, start_preprocessor_node, start_retriever, start_reader
|
11 |
from utils.ui import reset_results, set_initial_state
|
12 |
import pandas as pd
|
|
|
13 |
|
14 |
# Whether the file upload should be enabled or not
|
15 |
DISABLE_FILE_UPLOAD = bool(os.getenv("DISABLE_FILE_UPLOAD"))
|
16 |
-
|
17 |
# Define a function to handle file uploads
|
18 |
def upload_files():
|
19 |
uploaded_files = st.sidebar.file_uploader(
|
@@ -49,28 +49,41 @@ def process_file(data_file, preprocesor, document_store):
|
|
49 |
|
50 |
try:
|
51 |
args = parser.parse_args()
|
52 |
-
#session_state = st.session_state
|
53 |
preprocesor = start_preprocessor_node()
|
54 |
document_store = start_document_store(type=args.store)
|
55 |
retriever = start_retriever(document_store)
|
56 |
reader = start_reader()
|
57 |
st.set_page_config(
|
58 |
-
page_title="
|
59 |
layout="centered",
|
60 |
-
page_icon
|
61 |
menu_items={
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
|
|
66 |
st.sidebar.image("ml_logo.png", use_column_width=True)
|
67 |
|
68 |
# Sidebar for Task Selection
|
69 |
st.sidebar.header('Options:')
|
70 |
-
task_selection = st.sidebar.radio('Select the task:', ['Extractive', 'Generative'])
|
71 |
|
72 |
-
|
73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
74 |
|
75 |
set_initial_state()
|
76 |
|
@@ -138,9 +151,12 @@ try:
|
|
138 |
"π An error occurred reading the results. Is the document store working?"
|
139 |
)
|
140 |
except Exception as e:
|
141 |
-
|
142 |
-
|
143 |
-
|
|
|
|
|
|
|
144 |
# Display results
|
145 |
if (st.session_state.results_extractive or st.session_state.results_generative) and run_query:
|
146 |
|
|
|
1 |
+
from operator import index
|
2 |
import streamlit as st
|
3 |
import logging
|
4 |
import os
|
|
|
10 |
from utils.haystack import start_document_store, query, initialize_pipeline, start_preprocessor_node, start_retriever, start_reader
|
11 |
from utils.ui import reset_results, set_initial_state
|
12 |
import pandas as pd
|
13 |
+
import haystack
|
14 |
|
15 |
# Whether the file upload should be enabled or not
|
16 |
DISABLE_FILE_UPLOAD = bool(os.getenv("DISABLE_FILE_UPLOAD"))
|
|
|
17 |
# Define a function to handle file uploads
|
18 |
def upload_files():
|
19 |
uploaded_files = st.sidebar.file_uploader(
|
|
|
49 |
|
50 |
try:
|
51 |
args = parser.parse_args()
|
|
|
52 |
preprocesor = start_preprocessor_node()
|
53 |
document_store = start_document_store(type=args.store)
|
54 |
retriever = start_retriever(document_store)
|
55 |
reader = start_reader()
|
56 |
st.set_page_config(
|
57 |
+
page_title="MLReplySearch",
|
58 |
layout="centered",
|
59 |
+
page_icon=":shark:",
|
60 |
menu_items={
|
61 |
+
'Get Help': 'https://www.extremelycoolapp.com/help',
|
62 |
+
'Report a bug': "https://www.extremelycoolapp.com/bug",
|
63 |
+
'About': "# This is a header. This is an *extremely* cool app!"
|
64 |
+
}
|
65 |
+
)
|
66 |
st.sidebar.image("ml_logo.png", use_column_width=True)
|
67 |
|
68 |
# Sidebar for Task Selection
|
69 |
st.sidebar.header('Options:')
|
|
|
70 |
|
71 |
+
# OpenAI Key Input
|
72 |
+
openai_key = st.sidebar.text_input("Enter OpenAI Key:", type="password")
|
73 |
+
|
74 |
+
if openai_key:
|
75 |
+
task_options = ['Extractive', 'Generative']
|
76 |
+
else:
|
77 |
+
task_options = ['Extractive']
|
78 |
+
|
79 |
+
task_selection = st.sidebar.radio('Select the task:', task_options)
|
80 |
+
|
81 |
+
# Check the task and initialize pipeline accordingly
|
82 |
+
if task_selection == 'Extractive':
|
83 |
+
pipeline_extractive = initialize_pipeline("extractive", document_store, retriever, reader)
|
84 |
+
elif task_selection == 'Generative' and openai_key: # Check for openai_key to ensure user has entered it
|
85 |
+
pipeline_rag = initialize_pipeline("rag", document_store, retriever, reader, openai_key=openai_key)
|
86 |
+
|
87 |
|
88 |
set_initial_state()
|
89 |
|
|
|
151 |
"π An error occurred reading the results. Is the document store working?"
|
152 |
)
|
153 |
except Exception as e:
|
154 |
+
if "API key is invalid" in str(e):
|
155 |
+
logging.exception(e)
|
156 |
+
st.error("π incorrect API key provided. You can find your API key at https://platform.openai.com/account/api-keys.")
|
157 |
+
else:
|
158 |
+
logging.exception(e)
|
159 |
+
st.error("π An error occurred during the request.")
|
160 |
# Display results
|
161 |
if (st.session_state.results_extractive or st.session_state.results_generative) and run_query:
|
162 |
|
utils/config.py
CHANGED
@@ -7,9 +7,7 @@ load_dotenv()
|
|
7 |
parser = argparse.ArgumentParser(description='This app lists animals')
|
8 |
|
9 |
document_store_choices = ('inmemory', 'weaviate', 'milvus', 'opensearch')
|
10 |
-
task_choices = ('extractive', 'rag')
|
11 |
parser.add_argument('--store', choices=document_store_choices, default='inmemory', help='DocumentStore selection (default: %(default)s)')
|
12 |
-
#parser.add_argument('--task', choices=task_choices, default='rag', help='Task selection (default: %(default)s)')
|
13 |
parser.add_argument('--name', default="My Search App")
|
14 |
|
15 |
model_configs = {
|
|
|
7 |
parser = argparse.ArgumentParser(description='This app lists animals')
|
8 |
|
9 |
document_store_choices = ('inmemory', 'weaviate', 'milvus', 'opensearch')
|
|
|
10 |
parser.add_argument('--store', choices=document_store_choices, default='inmemory', help='DocumentStore selection (default: %(default)s)')
|
|
|
11 |
parser.add_argument('--name', default="My Search App")
|
12 |
|
13 |
model_configs = {
|
utils/haystack.py
CHANGED
@@ -32,6 +32,7 @@ def start_document_store(type: str):
|
|
32 |
print('initializing document store')
|
33 |
if type == 'inmemory':
|
34 |
document_store = InMemoryDocumentStore(use_bm25=True, embedding_dim=384)
|
|
|
35 |
documents = [
|
36 |
{
|
37 |
'content': "Pi is a super dog",
|
@@ -42,7 +43,8 @@ def start_document_store(type: str):
|
|
42 |
'meta': {'name': "siemens.txt"}
|
43 |
},
|
44 |
]
|
45 |
-
|
|
|
46 |
elif type == 'opensearch':
|
47 |
document_store = OpenSearchDocumentStore(scheme = document_store_configs['OPENSEARCH_SCHEME'],
|
48 |
username = document_store_configs['OPENSEARCH_USERNAME'],
|
@@ -94,10 +96,10 @@ def start_haystack_extractive(_document_store: BaseDocumentStore, _retriever: Em
|
|
94 |
return pipe
|
95 |
|
96 |
@st.cache_resource(show_spinner=False)
|
97 |
-
def start_haystack_rag(_document_store: BaseDocumentStore, _retriever: EmbeddingRetriever):
|
98 |
prompt_node = PromptNode(default_prompt_template="deepset/question-answering",
|
99 |
model_name_or_path=model_configs['GENERATIVE_MODEL'],
|
100 |
-
api_key=
|
101 |
pipe = Pipeline()
|
102 |
|
103 |
pipe.add_node(component=_retriever, name="Retriever", inputs=["Query"])
|
@@ -111,8 +113,8 @@ def query(_pipeline, question):
|
|
111 |
results = _pipeline.run(question, params=params)
|
112 |
return results
|
113 |
|
114 |
-
def initialize_pipeline(task, document_store, retriever, reader):
|
115 |
if task == 'extractive':
|
116 |
return start_haystack_extractive(document_store, retriever, reader)
|
117 |
elif task == 'rag':
|
118 |
-
return start_haystack_rag(document_store, retriever)
|
|
|
32 |
print('initializing document store')
|
33 |
if type == 'inmemory':
|
34 |
document_store = InMemoryDocumentStore(use_bm25=True, embedding_dim=384)
|
35 |
+
'''
|
36 |
documents = [
|
37 |
{
|
38 |
'content': "Pi is a super dog",
|
|
|
43 |
'meta': {'name': "siemens.txt"}
|
44 |
},
|
45 |
]
|
46 |
+
document_store.write_documents(documents)
|
47 |
+
'''
|
48 |
elif type == 'opensearch':
|
49 |
document_store = OpenSearchDocumentStore(scheme = document_store_configs['OPENSEARCH_SCHEME'],
|
50 |
username = document_store_configs['OPENSEARCH_USERNAME'],
|
|
|
96 |
return pipe
|
97 |
|
98 |
@st.cache_resource(show_spinner=False)
|
99 |
+
def start_haystack_rag(_document_store: BaseDocumentStore, _retriever: EmbeddingRetriever, openai_key):
|
100 |
prompt_node = PromptNode(default_prompt_template="deepset/question-answering",
|
101 |
model_name_or_path=model_configs['GENERATIVE_MODEL'],
|
102 |
+
api_key=openai_key)
|
103 |
pipe = Pipeline()
|
104 |
|
105 |
pipe.add_node(component=_retriever, name="Retriever", inputs=["Query"])
|
|
|
113 |
results = _pipeline.run(question, params=params)
|
114 |
return results
|
115 |
|
116 |
+
def initialize_pipeline(task, document_store, retriever, reader, openai_key = ""):
|
117 |
if task == 'extractive':
|
118 |
return start_haystack_extractive(document_store, retriever, reader)
|
119 |
elif task == 'rag':
|
120 |
+
return start_haystack_rag(document_store, retriever, openai_key)
|