DeepVen commited on
Commit
9037361
1 Parent(s): 1ab2494

Upload 11 files

Browse files
Files changed (11) hide show
  1. .streamlit/config.toml +18 -0
  2. Dockerfile +31 -0
  3. Index.py +326 -0
  4. README.md +5 -4
  5. app.py +20 -0
  6. extractor.py +94 -0
  7. hallucinator.py +19 -0
  8. main.py +85 -0
  9. requirements.txt +12 -0
  10. streamapp.py +317 -0
  11. test.py +12 -0
.streamlit/config.toml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [theme] # You have to add this line
2
+
3
+ #primaryColor = '#FF8C02' # Bright Orange
4
+
5
+ #secondaryColor = '#FF8C02' # Bright Orange
6
+
7
+ #backgroundColor = '#00325B' # Dark Blue
8
+
9
+ #secondaryBackgroundColor = '#55B2FF' # Lighter Blue
10
+
11
+
12
+ primaryColor="#ff4b4b"
13
+ backgroundColor="#00325B"
14
+ secondaryBackgroundColor="#262730"
15
+ textColor="#fafafa"
16
+ font="monospace"
17
+
18
+
Dockerfile ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use the official Python 3.9 image
2
+ FROM python:3.9
3
+
4
+ # Set the working directory to /code
5
+ WORKDIR /code
6
+
7
+ # Copy the current directory contents into the container at /code
8
+ COPY ./requirements.txt /code/requirements.txt
9
+
10
+
11
+
12
+ # Set up a new user named "user" with user ID 1000
13
+ RUN useradd -m -u 1000 user
14
+ # Switch to the "user" user
15
+ USER user
16
+ # Set home to the user's home directory
17
+ ENV HOME=/home/user \
18
+ PATH=/home/user/.local/bin:$PATH
19
+
20
+ # Install requirements.txt
21
+ RUN pip install --no-cache-dir --upgrade --user -r /code/requirements.txt
22
+
23
+ # Set the working directory to the user's home directory
24
+ WORKDIR $HOME/app
25
+
26
+ EXPOSE 6060
27
+
28
+ # Copy the current directory contents into the container at $HOME/app setting the owner to the user
29
+ COPY --chown=user . $HOME/app
30
+
31
+ CMD ["streamlit", "run", "demo_app/main.py", "--server.port", "7860"]
Index.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ import os
3
+ import json
4
+ import pandas as pd
5
+ import time
6
+
7
+ import phoenix as px
8
+ from phoenix.trace.langchain import OpenInferenceTracer, LangChainInstrumentor
9
+
10
+
11
+ from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models
12
+ from langchain.chains.question_answering import load_qa_chain
13
+ from langchain import HuggingFaceHub
14
+ from langchain.prompts import PromptTemplate
15
+
16
+ from langchain.chains import RetrievalQA
17
+ from langchain.callbacks import StdOutCallbackHandler
18
+
19
+ #from langchain.retrievers import KNNRetriever
20
+ from langchain.storage import LocalFileStore
21
+ from langchain.embeddings import CacheBackedEmbeddings
22
+ from langchain.vectorstores import FAISS
23
+
24
+
25
+ from langchain.document_loaders import WebBaseLoader
26
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
27
+
28
+ # from langchain import HuggingFaceHub
29
+ # from langchain.prompts import PromptTemplate
30
+ # from langchain.chains import LLMChain
31
+ # from txtai.embeddings import Embeddings
32
+ # from txtai.pipeline import Extractor
33
+
34
+ # import pandas as pd
35
+ # import sqlite3
36
+ # import os
37
+
38
+ # NOTE - we configure docs_url to serve the interactive Docs at the root path
39
+ # of the app. This way, we can use the docs as a landing page for the app on Spaces.
40
+ app = FastAPI(docs_url="/")
41
+
42
+ #phoenix setup
43
+ session = px.launch_app()
44
+ # If no exporter is specified, the tracer will export to the locally running Phoenix server
45
+ tracer = OpenInferenceTracer()
46
+ # If no tracer is specified, a tracer is constructed for you
47
+ LangChainInstrumentor(tracer).instrument()
48
+ print(session.url)
49
+
50
+
51
+ os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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"
52
+
53
+ # embedding cache
54
+ store = LocalFileStore("./cache/")
55
+
56
+ # define embedder
57
+ core_embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
58
+ embedder = CacheBackedEmbeddings.from_bytes_store(core_embeddings_model, store)
59
+
60
+ # define llm
61
+ llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
62
+ #llm=HuggingFaceHub(repo_id="gpt2", model_kwargs={"temperature":1, "max_length":1000000})
63
+ handler = StdOutCallbackHandler()
64
+
65
+ # set global variable
66
+ vectorstore = None
67
+ retriever = None
68
+
69
+
70
+ def initialize_vectorstore():
71
+
72
+ webpage_loader = WebBaseLoader("https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us").load()
73
+ webpage_chunks = _text_splitter(webpage_loader)
74
+
75
+ global vectorstore
76
+ global retriever
77
+
78
+ # store embeddings in vector store
79
+ vectorstore = FAISS.from_documents(webpage_chunks, embedder)
80
+ print("vector store initialized with sample doc")
81
+
82
+ # instantiate a retriever
83
+ retriever = vectorstore.as_retriever()
84
+
85
+
86
+ def _text_splitter(doc):
87
+ text_splitter = RecursiveCharacterTextSplitter(
88
+ chunk_size=600,
89
+ chunk_overlap=50,
90
+ length_function=len,
91
+ )
92
+ return text_splitter.transform_documents(doc)
93
+
94
+ def _load_docs(path: str):
95
+ load_doc = WebBaseLoader(path).load()
96
+ doc = _text_splitter(load_doc)
97
+ return doc
98
+
99
+
100
+ @app.get("/index/")
101
+ def get_domain_file_path(file_path: str):
102
+ print("file_path " ,file_path)
103
+
104
+ webpage_loader = _load_docs(file_path)
105
+
106
+ webpage_chunks = _text_splitter(webpage_loader)
107
+
108
+ # store embeddings in vector store
109
+ vectorstore.add_documents(webpage_chunks)
110
+
111
+ return "document loaded to vector store successfully!!"
112
+
113
+
114
+ def _prompt(question):
115
+ return f"""Answer following question using only the context below. Say 'Could not find answer with provided context' when question can't be answered.
116
+ Question: {question}
117
+ Context: """
118
+
119
+
120
+
121
+ class BasePromptContext:
122
+ def __init__(self):
123
+ self.variables_list = ["question","answer","context"]
124
+ self.base_template = """Please act as an impartial judge and evaluate the quality of the provided answer which attempts to answer the provided question based on a provided context.
125
+ And you'll need to submit your grading for the correctness, comprehensiveness and readability of the answer, using JSON format with the 2 items in parenthesis:
126
+ ("score": [your score number for the correctness of the answer], "reasoning": [your one line step by step reasoning about the correctness of the answer])
127
+ Below is your grading rubric:
128
+ - Correctness: If the answer correctly answer the question, below are the details for different scores:
129
+ - Score 0: the answer is completely incorrect, doesn’t mention anything about the question or is completely contrary to the correct answer.
130
+ - For example, when asked “How to terminate a databricks cluster”, the answer is empty string, or content that’s completely irrelevant, or sorry I don’t know the answer.
131
+ - Score 4: the answer provides some relevance to the question and answer one aspect of the question correctly.
132
+ - Example:
133
+ - Question: How to terminate a databricks cluster
134
+ - Answer: Databricks cluster is a cloud-based computing environment that allows users to process big data and run distributed data processing tasks efficiently.
135
+ - Or answer: In the Databricks workspace, navigate to the "Clusters" tab. And then this is a hard question that I need to think more about it
136
+ - Score 7: the answer mostly answer the question but is missing or hallucinating on one critical aspect.
137
+ - Example:
138
+ - Question: How to terminate a databricks cluster”
139
+ - Answer: “In the Databricks workspace, navigate to the "Clusters" tab.
140
+ Find the cluster you want to terminate from the list of active clusters.
141
+ And then you’ll find a button to terminate all clusters at once”
142
+ - Score 10: the answer correctly answer the question and not missing any major aspect
143
+ - Example:
144
+ - Question: How to terminate a databricks cluster
145
+ - Answer: In the Databricks workspace, navigate to the "Clusters" tab.
146
+ Find the cluster you want to terminate from the list of active clusters.
147
+ Click on the down-arrow next to the cluster name to open the cluster details.
148
+ Click on the "Terminate" button. A confirmation dialog will appear. Click "Terminate" again to confirm the action.”
149
+ Provided question:
150
+ {question}
151
+ Provided answer:
152
+ {answer}
153
+ Provided context:
154
+ {context}
155
+ Please provide your grading for the correctness and explain you gave the particular grading"""
156
+
157
+
158
+ class Evaluater:
159
+ def __init__(self, item):
160
+ self.question = item["question"]
161
+ self.answer = item["answer"]
162
+ #self.domain = item["domain"]
163
+ self.context = item["context"]
164
+ self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
165
+
166
+ def get_prompt_template(self):
167
+ prompt = BasePromptContext()
168
+ template = prompt.base_template
169
+ varialbles = prompt.variables_list
170
+ eval_template = PromptTemplate(input_variables=varialbles, template=template)
171
+ return eval_template
172
+
173
+ def evaluate(self):
174
+ prompt = self.get_prompt_template().format(question = self.question, answer = self.answer, context = self.context)
175
+ score = self.llm(prompt)
176
+ return score
177
+
178
+ # Create extractor instance
179
+ def _create_evaluation_scenario(item):
180
+ score = Evaluater(item).evaluate()
181
+ return score
182
+
183
+
184
+ class HallucinatePromptContext:
185
+ def __init__(self):
186
+ self.variables_list = ["query","answer","context"]
187
+ self.base_template = """In this task, you will be presented with a query, a reference text and an answer. The answer is
188
+
189
+ generated to the question based on the reference text. The answer may contain false information, you
190
+
191
+ must use the reference text to determine if the answer to the question contains false information,
192
+
193
+ if the answer is a hallucination of facts. Your objective is to determine whether the reference text
194
+
195
+ contains factual information and is not a hallucination. A 'hallucination' in this context refers to
196
+
197
+ an answer that is not based on the reference text or assumes information that is not available in
198
+
199
+ the reference text. Your response should be a single word: either "factual" or "hallucinated", and
200
+
201
+ it should not include any other text or characters. "hallucinated" indicates that the answer
202
+
203
+ provides factually inaccurate information to the query based on the reference text. "factual"
204
+
205
+ indicates that the answer to the question is correct relative to the reference text, and does not
206
+
207
+ contain made up information. Please read the query and reference text carefully before determining
208
+
209
+ your response.
210
+
211
+
212
+
213
+ # Query: {query}
214
+
215
+ # Reference text: {context}
216
+
217
+ # Answer: {answer}
218
+
219
+ Is the answer above factual or hallucinated based on the query and reference text?"""
220
+
221
+
222
+ class HallucinatonEvaluater:
223
+ def __init__(self, item):
224
+ self.question = item["question"]
225
+ self.answer = item["answer"]
226
+ #self.domain = item["domain"]
227
+ self.context = item["context"]
228
+ self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
229
+
230
+ def get_prompt_template(self):
231
+ prompt = HallucinatePromptContext()
232
+ template = prompt.base_template
233
+ varialbles = prompt.variables_list
234
+ eval_template = PromptTemplate(input_variables=varialbles, template=template)
235
+ return eval_template
236
+
237
+ def evaluate(self):
238
+ prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
239
+ score = self.llm(prompt)
240
+ return score
241
+
242
+ # Create extractor instance
243
+ def _create_evaluation_scenario(item):
244
+ score = Evaluater(item).evaluate()
245
+ return score
246
+
247
+ # Create extractor instance
248
+ def _create_hallucination_scenario(item):
249
+ score = HallucinatonEvaluater(item).evaluate()
250
+ return score
251
+
252
+
253
+ @app.get("/rag")
254
+ def rag( question: str, evaluate: bool):
255
+
256
+ chain = RetrievalQA.from_chain_type(
257
+ llm=llm,
258
+ retriever=retriever,
259
+ callbacks=[handler],
260
+ return_source_documents=True
261
+ )
262
+
263
+ #response = chain("how tredence brought good insight?")
264
+ response = chain(question)
265
+ print(response["result"])
266
+
267
+ time.sleep(4)
268
+
269
+ df = px.active_session().get_spans_dataframe()
270
+ print(px.active_session())
271
+ print(px.active_session().get_spans_dataframe())
272
+ print(df.count())
273
+ df_sorted = df.sort_values(by='end_time',ascending=False)
274
+
275
+ model_input = json.loads(df_sorted[df_sorted["name"] == "LLMChain"]["attributes.input.value"][0])
276
+ context = model_input["context"]
277
+
278
+ print(context)
279
+
280
+ if evaluate:
281
+ score = _create_evaluation_scenario({
282
+ "question": question,
283
+ "answer": response['result'],
284
+ "context": context
285
+ })
286
+ else:
287
+ score = "Evaluation is Turned OFF"
288
+
289
+ return {"question": question, "answer": response['result'], "context": context, "score": score}
290
+
291
+
292
+ initialize_vectorstore()
293
+
294
+
295
+ @app.get("/trace")
296
+ def trace():
297
+ df = px.active_session().get_spans_dataframe().fillna('')
298
+ return df
299
+
300
+
301
+ @app.get("/hallucinate")
302
+ def trace(question: str, answer: str, context: str):
303
+ hallucination_score = _create_hallucination_scenario({
304
+ "question": question,
305
+ "answer": answer,
306
+ "context": context
307
+ }
308
+ )
309
+
310
+ return {"hallucination_score": hallucination_score}
311
+
312
+
313
+
314
+
315
+ '''
316
+ #import getpass
317
+ from pyngrok import ngrok, conf
318
+ #print("Enter your authtoken, which can be copied from https://dashboard.ngrok.com/auth")
319
+ conf.get_default().auth_token="2WJNWULs5bCOyJnV24WQYJEKod3_YQUbM5EGCp8sgE4aQvzi"
320
+ port = 37689
321
+ # Open a ngrok tunnel to the HTTP server
322
+ conf.get_default().monitor_thread = False
323
+ public_url = ngrok.connect(port).public_url
324
+ print(" * ngrok tunnel \"{}\" -> \"http://127.0.0.1:{}\"".format(public_url, port))
325
+ '''
326
+
README.md CHANGED
@@ -1,10 +1,11 @@
1
  ---
2
- title: Streamlit
3
- emoji: 🏃
4
- colorFrom: indigo
5
- colorTo: pink
6
  sdk: docker
7
  pinned: false
 
8
  ---
9
 
10
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Text Generation
3
+ emoji: 🌍
4
+ colorFrom: green
5
+ colorTo: yellow
6
  sdk: docker
7
  pinned: false
8
+ license: mit
9
  ---
10
 
11
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ from transformers import pipeline
3
+
4
+
5
+ # NOTE - we configure docs_url to serve the interactive Docs at the root path
6
+ # of the app. This way, we can use the docs as a landing page for the app on Spaces.
7
+ app = FastAPI(docs_url="/")
8
+
9
+ pipe = pipeline("text2text-generation", model="google/flan-t5-small")
10
+
11
+
12
+ @app.get("/generate")
13
+ def generate(text: str):
14
+ """
15
+ Using the text2text-generation pipeline from `transformers`, generate text
16
+ from the given input text. The model used is `google/flan-t5-small`, which
17
+ can be found [here](https://huggingface.co/google/flan-t5-small).
18
+ """
19
+ output = pipe(text)
20
+ return {"output": output[0]["generated_text"]}
extractor.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+
3
+ # from transformers import pipeline
4
+ from txtai.embeddings import Embeddings
5
+ from txtai.pipeline import Extractor
6
+ from langchain.document_loaders import WebBaseLoader
7
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
8
+
9
+ # NOTE - we configure docs_url to serve the interactive Docs at the root path
10
+ # of the app. This way, we can use the docs as a landing page for the app on Spaces.
11
+ app = FastAPI(docs_url="/")
12
+
13
+ # Create embeddings model with content support
14
+ embeddings = Embeddings(
15
+ {"path": "sentence-transformers/all-MiniLM-L6-v2", "content": True}
16
+ )
17
+
18
+
19
+ # Create extractor instance
20
+ # extractor = Extractor(embeddings, "google/flan-t5-base")
21
+
22
+
23
+ def _stream(dataset, limit, index: int = 0):
24
+ for row in dataset:
25
+ yield (index, row.page_content, None)
26
+ index += 1
27
+
28
+ if index >= limit:
29
+ break
30
+
31
+
32
+ def _max_index_id(path):
33
+ db = sqlite3.connect(path)
34
+
35
+ table = "sections"
36
+ df = pd.read_sql_query(f"select * from {table}", db)
37
+ return {"max_index": df["indexid"].max()}
38
+
39
+
40
+ def _prompt(question):
41
+ return f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
42
+ Question: {question}
43
+ Context: """
44
+
45
+
46
+ async def _search(query, extractor, question=None):
47
+ # Default question to query if empty
48
+ if not question:
49
+ question = query
50
+
51
+ return extractor([("answer", query, _prompt(question), False)])[0][1]
52
+
53
+
54
+ def _text_splitter(doc):
55
+ text_splitter = RecursiveCharacterTextSplitter(
56
+ chunk_size=500,
57
+ chunk_overlap=50,
58
+ length_function=len,
59
+ )
60
+ return text_splitter.transform_documents(doc)
61
+
62
+
63
+ def _load_docs(path: str):
64
+ load_doc = WebBaseLoader(path).load()
65
+ doc = _text_splitter(load_doc)
66
+ return doc
67
+
68
+
69
+ async def _upsert_docs(doc):
70
+ max_index = _max_index_id("index/documents")
71
+ embeddings.upsert(_stream(doc, 500, max_index["max_index"]))
72
+ embeddings.save("index")
73
+
74
+ return embeddings
75
+
76
+
77
+ @app.put("/rag/{path}")
78
+ async def get_doc_path(path: str):
79
+ return path
80
+
81
+
82
+ @app.get("/rag")
83
+ async def rag(question: str):
84
+ # question = "what is the document about?"
85
+ embeddings.load("index")
86
+ path = await get_doc_path(path)
87
+ doc = _load_docs(path)
88
+ embeddings = _upsert_docs(doc)
89
+
90
+ # Create extractor instance
91
+ extractor = Extractor(embeddings, "google/flan-t5-base")
92
+ answer = await _search(question, extractor)
93
+ # print(question, answer)
94
+ return {answer}
hallucinator.py ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ class HallucinatonEvaluater:
2
+ def __init__(self, item):
3
+ self.question = item["question"]
4
+ self.answer = item["answer"]
5
+ #self.domain = item["domain"]
6
+ self.context = item["context"]
7
+ self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
8
+
9
+ def get_prompt_template(self):
10
+ prompt = HallucinatePromptContext()
11
+ template = prompt.base_template
12
+ varialbles = prompt.variables_list
13
+ eval_template = PromptTemplate(input_variables=varialbles, template=template)
14
+ return eval_template
15
+
16
+ def evaluate(self):
17
+ prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
18
+ score = self.llm(prompt)
19
+ return score
main.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from fastapi import FastAPI
2
+ from txtai.embeddings import Embeddings
3
+ from txtai.pipeline import Extractor
4
+ import os
5
+ from langchain import HuggingFaceHub
6
+ from langchain.prompts import PromptTemplate
7
+ from langchain.chains import LLMChain
8
+
9
+ # from transformers import pipeline
10
+
11
+ # NOTE - we configure docs_url to serve the interactive Docs at the root path
12
+ # of the app. This way, we can use the docs as a landing page for the app on Spaces.
13
+ app = FastAPI(docs_url="/")
14
+
15
+ # @app.get("/generate")
16
+ # def generate(text: str):
17
+ # """
18
+ # Using the text2text-generation pipeline from `transformers`, generate text
19
+ # from the given input text. The model used is `google/flan-t5-small`, which
20
+ # can be found [here](https://huggingface.co/google/flan-t5-small).
21
+ # """
22
+ # output = pipe(text)
23
+ # return {"output": output[0]["generated_text"]}
24
+
25
+
26
+ def _check_if_db_exists(db_path: str) -> bool:
27
+ return os.path.exists(db_path)
28
+
29
+
30
+ def _load_embeddings_from_db(
31
+ db_present: bool,
32
+ domain: str,
33
+ path: str = "sentence-transformers/all-MiniLM-L6-v2",
34
+ ):
35
+ # Create embeddings model with content support
36
+ embeddings = Embeddings({"path": path, "content": True})
37
+ # if Vector DB is not present
38
+ if not db_present:
39
+ return embeddings
40
+ else:
41
+ if domain == "":
42
+ embeddings.load("index") # change this later
43
+ else:
44
+ print(3)
45
+ embeddings.load(f"index/{domain}")
46
+ return embeddings
47
+
48
+
49
+ def _prompt(question):
50
+ return f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
51
+ Question: {question}
52
+ Context: """
53
+
54
+
55
+ def _search(query, extractor, question=None):
56
+ # Default question to query if empty
57
+ if not question:
58
+ question = query
59
+
60
+ # template = f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
61
+ # Question: {question}
62
+ # Context: """
63
+
64
+ # prompt = PromptTemplate(template=template, input_variables=["question"])
65
+ # llm_chain = LLMChain(prompt=prompt, llm=extractor)
66
+
67
+ # return {"question": question, "answer": llm_chain.run(question)}
68
+ return extractor([("answer", query, _prompt(question), False)])[0][1]
69
+
70
+
71
+ @app.get("/rag")
72
+ def rag(domain: str, question: str):
73
+ db_exists = _check_if_db_exists(db_path=f"{os.getcwd()}\index\{domain}\documents")
74
+ print(db_exists)
75
+ # if db_exists:
76
+ embeddings = _load_embeddings_from_db(db_exists, domain)
77
+ # Create extractor instance
78
+ extractor = Extractor(embeddings, "google/flan-t5-base")
79
+ # llm = HuggingFaceHub(
80
+ # repo_id="google/flan-t5-xxl",
81
+ # model_kwargs={"temperature": 1, "max_length": 1000000},
82
+ # )
83
+ # else:
84
+ answer = _search(question, extractor)
85
+ return {"question": question, "answer": answer}
requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi==0.74.*
2
+ requests==2.27.*
3
+ uvicorn[standard]==0.17.*
4
+ sentencepiece==0.1.*
5
+ torch==1.12.*
6
+ transformers==4.*
7
+ langchain==0.0.301
8
+ arize-phoenix
9
+ huggingface_hub
10
+ faiss-cpu
11
+ bs4==0.0.1
12
+ streamlit
streamapp.py ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import pandas as pd
4
+ import time
5
+
6
+ #from hallucinator import HallucinatonEvaluater
7
+
8
+ from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models
9
+ from langchain.chains.question_answering import load_qa_chain
10
+ from langchain import HuggingFaceHub
11
+ from langchain.prompts import PromptTemplate
12
+
13
+ from langchain.chains import RetrievalQA
14
+ from langchain.callbacks import StdOutCallbackHandler
15
+
16
+ #from langchain.retrievers import KNNRetriever
17
+ from langchain.storage import LocalFileStore
18
+ from langchain.embeddings import CacheBackedEmbeddings
19
+ from langchain.vectorstores import FAISS
20
+
21
+
22
+ from langchain.document_loaders import WebBaseLoader
23
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
24
+
25
+
26
+ import numpy as np
27
+ import streamlit as st
28
+ import pandas as pd
29
+ # from sklearn import datasets
30
+ # from sklearn.ensemble import RandomForestClassifier
31
+
32
+
33
+ tab1, tab2 = st.tabs(["📈 RAG", "🗃 FactVsHallucinate"])
34
+
35
+
36
+
37
+
38
+ os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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"
39
+
40
+ # embedding cache
41
+ #store = LocalFileStore("./cache/")
42
+
43
+ # define embedder
44
+ embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
45
+ #embedder=HuggingFaceHub(repo_id="sentence-transformers/all-mpnet-base-v2")
46
+ #embedder = CacheBackedEmbeddings.from_bytes_store(core_embeddings_model, store)
47
+
48
+ # define llm
49
+ llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
50
+ #llm=HuggingFaceHub(repo_id="gpt2", model_kwargs={"temperature":1, "max_length":1000000})
51
+ handler = StdOutCallbackHandler()
52
+
53
+ # set global variable
54
+ # vectorstore = None
55
+ # retriever = None
56
+
57
+
58
+ class HallucinatePromptContext:
59
+ def __init__(self):
60
+ self.variables_list = ["query","answer","context"]
61
+ self.base_template = """In this task, you will be presented with a query, a reference text and an answer. The answer is
62
+ generated to the question based on the reference text. The answer may contain false information, you
63
+ must use the reference text to determine if the answer to the question contains false information,
64
+ if the answer is a hallucination of facts. Your objective is to determine whether the reference text
65
+ contains factual information and is not a hallucination. A 'hallucination' in this context refers to
66
+ an answer that is not based on the reference text or assumes information that is not available in
67
+ the reference text. Your response should be a single word: either "factual" or "hallucinated", and
68
+ it should not include any other text or characters. "hallucinated" indicates that the answer
69
+ provides factually inaccurate information to the query based on the reference text. "factual"
70
+ indicates that the answer to the question is correct relative to the reference text, and does not
71
+ contain made up information. Please read the query and reference text carefully before determining
72
+ your response.
73
+
74
+ # Query: {query}
75
+ # Reference text: {context}
76
+ # Answer: {answer}
77
+ Is the answer above factual or hallucinated based on the query and reference text?"""
78
+
79
+
80
+
81
+ class HallucinatonEvaluater:
82
+ def __init__(self, item):
83
+ self.question = item["question"]
84
+ self.answer = item["answer"]
85
+ #self.domain = item["domain"]
86
+ self.context = item["context"]
87
+ self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
88
+
89
+ def get_prompt_template(self):
90
+ prompt = HallucinatePromptContext()
91
+ template = prompt.base_template
92
+ varialbles = prompt.variables_list
93
+ eval_template = PromptTemplate(input_variables=varialbles, template=template)
94
+ return eval_template
95
+
96
+ def evaluate(self):
97
+ prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
98
+ score = self.llm(prompt)
99
+ return score
100
+
101
+
102
+
103
+
104
+ @st.cache_resource
105
+ def initialize_vectorstore():
106
+
107
+ webpage_loader = WebBaseLoader("https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us").load()
108
+ webpage_chunks = _text_splitter(webpage_loader)
109
+
110
+ global vectorstore
111
+ global retriever
112
+
113
+ # store embeddings in vector store
114
+ vectorstore = FAISS.from_documents(webpage_chunks, embedder)
115
+ print("vector store initialized with sample doc")
116
+
117
+ # instantiate a retriever
118
+ retriever = vectorstore.as_retriever()
119
+
120
+ return retriever
121
+
122
+
123
+ def _text_splitter(doc):
124
+ text_splitter = RecursiveCharacterTextSplitter(
125
+ chunk_size=600,
126
+ chunk_overlap=50,
127
+ length_function=len,
128
+ )
129
+ return text_splitter.transform_documents(doc)
130
+
131
+ def _load_docs(path: str):
132
+ load_doc = WebBaseLoader(path).load()
133
+ doc = _text_splitter(load_doc)
134
+ return doc
135
+
136
+
137
+
138
+
139
+
140
+
141
+ def rag_response(response):
142
+ st.markdown("""<hr style="height:10px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)
143
+ st.subheader('RAG response')
144
+ st.text_area(label="user query", value=response["query"], height=30)
145
+ st.text_area(label="RAG output", value=response["result"])
146
+ st.text_area(label="Augmented knowledge", value=response["source_documents"])
147
+
148
+ #st.button("Check Hallucination")
149
+
150
+
151
+
152
+
153
+
154
+ # Create extractor instance
155
+ def _create_hallucination_scenario(item):
156
+ score = HallucinatonEvaluater(item).evaluate()
157
+ return score
158
+
159
+ def hallu_eval(question: str, answer: str, context: str):
160
+ print("in hallu eval")
161
+ hallucination_score = _create_hallucination_scenario({
162
+ "question": question,
163
+ "answer": answer,
164
+ "context": context
165
+ }
166
+ )
167
+ print("got hallu score")
168
+ st.text_area(label="Hallucinated?", value=hallucination_score, height=30)
169
+ #return {"hallucination_score": hallucination_score}
170
+ #time.sleep(10)
171
+
172
+
173
+
174
+ # if 'clicked' not in st.session_state:
175
+ # print("set state to False")
176
+ # st.session_state.clicked = False
177
+
178
+
179
+ def click_button(response):
180
+ # print("set state to True")
181
+ # st.session_state.clicked = True
182
+
183
+ hallu_eval(response["query"], response["result"], "blah blah")
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+ #st.write(''' # RAG App''')
192
+
193
+ with tab1:
194
+
195
+ with st.form(" RAG with evaluation - scoring & hallucination "):
196
+ #tab1.subheader(''' # RAG App''')
197
+ retriever = initialize_vectorstore()
198
+
199
+ #print("lenght in tab1, ", len(vectorstore.serialize_to_bytes()))
200
+ options = ["true", "false"]
201
+ question = st.text_input(label="user question", value="", label_visibility="visible", disabled=False)
202
+ evaluate = st.selectbox(label="Evaluation",options=options, index=0, placeholder="Choose an option", disabled=False, label_visibility="visible")
203
+
204
+
205
+
206
+ if st.form_submit_button("RAG with evaluation"):
207
+ print("retrie ,", retriever)
208
+ chain = RetrievalQA.from_chain_type(
209
+ llm=llm,
210
+ retriever=retriever,
211
+ callbacks=[handler],
212
+ return_source_documents=True
213
+ )
214
+
215
+ #response = chain("how tredence brought good insight?")
216
+ response = chain(question)
217
+ print(response["result"])
218
+
219
+
220
+ rag_response(response)
221
+
222
+ click_button(response)
223
+
224
+
225
+
226
+
227
+
228
+ # if st.session_state.clicked:
229
+
230
+ # # The message and nested widget will remain on the page
231
+ # hallu_eval(response["query"], response["result"], "blah blah")
232
+
233
+
234
+ # print("in if for hallu")
235
+
236
+
237
+
238
+ with tab2:
239
+
240
+
241
+
242
+ with st.form(" LLM-aasisted evaluation of Hallucination"):
243
+
244
+
245
+ #print("lenght in tab2, ", len(vectorstore.serialize_to_bytes()))
246
+ question = st.text_input(label="question", value="", label_visibility="visible", disabled=False)
247
+ answer = st.text_input(label="answer", value="", label_visibility="visible", disabled=False)
248
+ context = st.text_input(label="context", value="", label_visibility="visible", disabled=False)
249
+
250
+
251
+ if st.form_submit_button("Evaluate"):
252
+ hallu_eval(question, answer, context)
253
+
254
+
255
+
256
+
257
+
258
+
259
+
260
+ def rag():
261
+ print("in rag")
262
+ options = ["true", "false"]
263
+ question = st.text_input(label="user question", value="", label_visibility="visible", disabled=False)
264
+ evaluate = st.selectbox(label="select evaluation",options=options, index=0, placeholder="Choose an option", disabled=False, label_visibility="visible")
265
+
266
+
267
+
268
+ if st.button("do RAG"):
269
+ chain = RetrievalQA.from_chain_type(
270
+ llm=llm,
271
+ retriever=retriever,
272
+ callbacks=[handler],
273
+ return_source_documents=True
274
+ )
275
+
276
+ #response = chain("how tredence brought good insight?")
277
+ response = chain(question)
278
+ print(response["result"])
279
+
280
+ # time.sleep(4)
281
+
282
+ # df = px.active_session().get_spans_dataframe()
283
+ # print(px.active_session())
284
+ # print(px.active_session().get_spans_dataframe())
285
+ # print(df.count())
286
+ # df_sorted = df.sort_values(by='end_time',ascending=False)
287
+
288
+ # model_input = json.loads(df_sorted[df_sorted["name"] == "LLMChain"]["attributes.input.value"][0])
289
+ # context = model_input["context"]
290
+
291
+ # print(context)
292
+
293
+ # if evaluate:
294
+ # score = _create_evaluation_scenario({
295
+ # "question": question,
296
+ # "answer": response['result'],
297
+ # "context": context
298
+ # })
299
+ # else:
300
+ # score = "Evaluation is Turned OFF"
301
+
302
+ # return {"question": question, "answer": response['result'], "context": context, "score": score}
303
+ rag_response(response)
304
+
305
+ # if st.button("click me"):
306
+ # click_button(response)
307
+
308
+ click = st.button("Do you want to see more?")
309
+ if click:
310
+ st.session_state.more_stuff = True
311
+
312
+ if st.session_state.more_stuff:
313
+ click_button(response)
314
+ #st.write("Doing more optional stuff")
315
+
316
+
317
+ return(response)
test.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ a= [
4
+ "Document(page_content='Key Benefits\\n\\nThe forecasting can help the clients in estimating the shortfalls in installs and activity of the new app, also aid in budget allocation to new app paid marketing\\n\\n\\n\\nResults\\n\\n\\n\\n\\n\\n\\n\\n\\n Enhanced the marketing efforts by 30% for successful migration of all existing app users to the new app portal\\n \\n\\n\\n\\n\\n\\n\\n\\n\\nTalk To Us\\n\\n\\n\\n\\n\\n\\nFull name *\\n\\n\\n\\nCompany\\n\\n\\n\\nEmail address *\\n\\n\\n\\nPhone number\\n\\n\\n\\nMessage\\n\\n\\n\\nConnect with\\n us', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n Forecasting App Installs for a Large Retailer in the US - Tredence\\n ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
5
+ "Document(page_content='Supply Chain\\n\\n\\nCustomer Analytics\\n\\n\\nCX Management\\n\\n\\nIndustry X.0\\n\\n\\nSupport Solutions\\n\\n\\n\\nSolutions\\n\\nSupply Chain Control Tower\\n\\n\\nTest and Learn Platform (TALP)\\n\\n\\nOn-shelf Availability (OSA)\\n\\n\\nCustomer Cosmos\\n\\n\\nRevenue Growth Management (RGM)\\n\\n\\nMLWorks\\n\\n\\nSustainability Analytics\\n\\n\\n\\n\\nTredence Studio\\n\\n\\n\\nBlogs\\n\\n\\n\\nClient Success\\n\\n\\n\\nLife at Tredence\\n\\n\\n\\nCareers\\n\\n\\n\\nContact us\\n\\n\\n\\n\\n\\nFollow us on\\n\\u2003\\n\\u2003\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nTerms of\\n Use\\n\\nPrivacy Policy\\n\\nCookies Policy\\n\\n\\n© Copyright . Tredence.Inc. All rights reserved\\n\\n\\n\\n \\n\\n\\n\\n\\n\\ntzCx9f4MNdCKSWQx6bxOR31y_2csI0A1Kb1rgsiOfdQ=.html', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n Forecasting App Installs for a Large Retailer in the US - Tredence\\n ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
6
+ "Document(page_content='Connect with\\n us\\n\\n\\n\\n\\n\\n\\n\\n\\nWe use cookies for the best user experience on our website, including to personalize content & offerings. By clicking “Accept Cookies” you agree to our use of cookies. For further details please visit our Privacy Policy and Cookies Policy\\n\\n\\n Accept cookies\\n \\n\\n Decline\\n \\n\\n\\n \\n\\n\\n\\nIndustries\\n\\nRetail\\n\\n\\nCPG\\n\\n\\nIndustrials\\n\\n\\nHealthcare\\n\\n\\nTMT\\n\\n\\nFinancial Services\\n\\n\\nTravel and Hospitality\\n\\n\\n\\nServices\\n\\nAI Consulting\\n\\n\\nData Engineering\\n\\n\\nSupply Chain\\n\\n\\nCustomer Analytics', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n Forecasting App Installs for a Large Retailer in the US - Tredence\\n ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
7
+ "Document(page_content='Summary\\n\\nClient was planning to bring in a new stand-alone app in the market which merges the different channels of the business. However, the marketing managers were not sure of the number of optimum marketing efforts that will be required to push the existing app users across BUs to install the new app. Tredence team aimed at forecasting the installs of the new app to help the client in planning the marketing budget and setting goals for the upcoming app migration in FY21\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nApproach', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n Forecasting App Installs for a Large Retailer in the US - Tredence\\n ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})"
8
+ ]
9
+
10
+
11
+ print(type(a[0]))
12
+ print(type(a[0]))