#%%
from tiktoken import get_encoding, encoding_for_model
from weaviate_interface import WeaviateClient, WhereFilter
from sentence_transformers import SentenceTransformer
from prompt_templates import question_answering_prompt_series, question_answering_system
from openai_interface import GPT_Turbo
from app_features import (convert_seconds, generate_prompt_series, search_result,
validate_token_threshold, load_content_cache, load_data,
expand_content)
from retrieval_evaluation import execute_evaluation, calc_hit_rate_scores
from llama_index.finetuning import EmbeddingQAFinetuneDataset
from weaviate_interface import WeaviateClient
from openai import BadRequestError
from reranker import ReRanker
from loguru import logger
import streamlit as st
from streamlit_option_menu import option_menu
import hydralit_components as hc
import sys
import json
import os, time, requests, re
from datetime import timedelta
import pathlib
import gdown
import tempfile
import base64
import shutil
def get_base64_of_bin_file(bin_file):
with open(bin_file, 'rb') as file:
data = file.read()
return base64.b64encode(data).decode()
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv('env'), override=True)
# I use a key that I increment each time I want to change a text_input
if 'key' not in st.session_state:
st.session_state.key = 0
# key = st.session_state['key']
if not pathlib.Path('models').exists():
os.mkdir('models')
# I should cache these things but no time left
# I put a file local.txt in my desktop models folder to find out if it's running online
we_are_online = not pathlib.Path("models/local.txt").exists()
we_are_not_online = not we_are_online
golden_dataset = EmbeddingQAFinetuneDataset.from_json("data/golden_100.json")
# shutil.rmtree("models/models") # remove it - I wanted to clear the space on streamlit online
## PAGE CONFIGURATION
st.set_page_config(page_title="Ask Impact Theory",
page_icon="assets/impact-theory-logo-only.png",
layout="wide",
initial_sidebar_state="collapsed",
menu_items={'Report a bug': "https://www.extremelycoolapp.com/bug"})
image = "https://is2-ssl.mzstatic.com/image/thumb/Music122/v4/bd/34/82/bd348260-314c-5898-26c0-bef2e0388ebe/source/1200x1200bb.png"
def add_bg_from_local(image_file):
bin_str = get_base64_of_bin_file(image_file)
page_bg_img = f'''
'''
st.markdown(page_bg_img, unsafe_allow_html=True)
# COMMENT: I tried to create a dropdown menu but it's harder than it looks, so I gave up
# https://discuss.streamlit.io/t/streamlit-option-menu-is-a-simple-streamlit-component-that-allows-users-to-select-a-single-item-from-a-list-of-options-in-a-menu/20514
# not great, but it works
# selected = option_menu("About", ["Improvements","This"], #"Main Menu", ["Home", 'Settings'],
# icons=['house', 'gear'],
# menu_icon="cast",
# default_index=1)
# # Custom HTML/CSS for the banner
# base64_img = get_base64_of_bin_file("assets/it_tom_bilyeu.png")
# banner_menu_html = f"""
#
#
#
'
# f'',
# unsafe_allow_html=True)
st.image(image, caption=title.split('|')[0], width=200, use_column_width=False)
# let's use all width for the content
st.write(hit['content'])
def get_answer(query, valid_response, GPTllm):
# generate LLM prompt
prompt = generate_prompt_series(query=query,
results=valid_response)
return GPTllm.get_chat_completion(prompt=prompt,
system_message='answer this question based on the podcast material',
temperature=0,
max_tokens=500,
stream=False,
show_response=False)
def reword_query(query, guest, model_name='llama2-13b-chat', response_processing=True):
""" Asks LLM to rewrite the query when the guest name is missing.
Args:
query (str): user query
guest (str): guest name
model_name (str, optional): name of a LLM model to be used
"""
# tags = {'llama2-13b-chat': {'start': '', 'end': '', 'instruction': '[INST]', 'system': '[SYS]'},
# 'gpt-3.5-turbo-0613': {'start': '<|startoftext|>', 'end': '', 'instruction': "```", 'system': ```}}
prompt_fields = {
"you_are":f"You are an expert in linguistics and semantics, analyzing the question asked by a user to a vector search system, \
and making sure that the question is well formulated and that the system can understand it.",
"your_task":f"Your task is to detect if the name of the guest ({guest}) is mentioned in the user's question, \
and if that is not the case, rewrite the question using the guest name, \
without changing the meaning of the question. \
Most of the time, the user will have used a pronoun to designate the guest, in which case, \
simply replace the pronoun with the guest name.",
"question":f"If the user mentions the guest name, ie {query}, just return his question as is. \
If the user does not mention the guest name, rewrite the question using the guest name.",
"final_instruction":f"Only regerate the requested rewritten question or the original, WITHOUT ANY COMMENT OR REPHRASING. \
Your answer must be as close as possible to the original question, \
and exactly identical, word for word, if the user mentions the guest name, i.e. {guest}.",
}
# prompt created by chatGPT :-)
# and Llama still outputs the original question and precedes the answer with 'rewritten question'
prompt_fields2 = {
"you_are": (
"You are an expert in linguistics and semantics. Your role is to analyze questions asked to a vector search system."
),
"your_task": (
f"Detect if the guest's FULL name, {guest}, is mentioned in the user's question. "
"If not, rewrite the question by replacing pronouns or indirect references with the guest's name." \
"If yes, return the original question as is, without any change at all, not even punctuation,"
"except a question mark that you MUST add if it's missing."
),
"question": (
f"Original question: '{query}'. "
"Rewrite this question to include the guest's FULL name if it's not already mentioned."
"The Only thing you can and MUST add is a question mark if it's missing."
),
"final_instruction": (
"Create a rewritten question or keep the original question as is. "
"Do not include any labels, titles, or additional text before or after the question."
"The Only thing you can and MUST add is a question mark if it's missing."
"Return a json object, with the key 'original_question' for the original question, \
and 'rewritten_question' for the rewritten question \
and 'changed' being True if you changed the answer, otherwise False."
),
}
if model_name == 'llama2-13b-chat':
# special tags are used:
# `` - start prompt tag
# `[INST], [/INST]` - Opening and closing model instruction tags
# `<<>>, <>` - Opening and closing system prompt tags
llama_prompt = """
[INST] <>
{you_are}
<>
{your_task}\n
```
\n\n
Question: {question}\n
{final_instruction} [/INST]
Answer:
"""
prompt = llama_prompt.format(**prompt_fields2)
hf_token = st.secrets['secrets']['LLAMA2_ENDPOINT_HF_TOKEN_chris']
# hf_token = st.secrets['secrets']['LLAMA2_ENDPOINT_HF_TOKEN']
hf_endpoint = st.secrets['secrets']['LLAMA2_ENDPOINT_UPLIMIT']
headers = {"Authorization": f"Bearer {hf_token}",
"Content-Type": "application/json",}
json_body = {
"inputs": prompt,
"parameters": {"max_new_tokens":400,
"repetition_penalty": 1.0,
"temperature":0.01}
}
response = requests.request("POST", hf_endpoint, headers=headers, data=json.dumps(json_body))
response = json.loads(response.content.decode("utf-8"))
# ^ will not process the badly formatted generated text, so we do it ourselves
if isinstance(response, dict) and 'error' in response:
print("Found error")
print(response)
# return {'error': response['error'], 'rewritten_question': query, 'changed': False, 'status': 'error'}
# I test this here otherwise it gets in col 2 or 1, which are too
# if reworded_query['status'] == 'error':
# st.write(f"Error in LLM response: 'error':{reworded_query['error']}")
# st.write("The LLM could not connect to the server. Please try again later.")
# st.stop()
return reword_query(query, guest, model_name='gpt-3.5-turbo-0613')
if response_processing:
if isinstance(response, list) and isinstance(response[0], dict) and 'generated_text' in response[0]:
print("Found generated text")
response0 = response[0]['generated_text']
pattern = r'\"(\w+)\":\s*(\".*?\"|\w+)'
matches = re.findall(pattern, response0)
# let's build a dictionary
result = {key: json.loads(value) if value.startswith("\"") else value for key, value in matches}
return result | {'status': 'success'}
else:
print("Found no answer")
return reword_query(query, guest, model_name='gpt-3.5-turbo-0613')
# return {'original_question': query, 'rewritten_question': query, 'changed': False, 'status': 'no properly formatted answer' }
else:
return response
# return response
# assert 'error' not in response, f"Error in LLM response: {response['error']}"
# assert 'generated_text' in response[0], f"Error in LLM response: {response}, no 'generated_text' field"
# # let's extract the rewritten question
# return response[0]['generated_text'] .split("Rewritten question: '")[-1][:-1]
else:
# assume openai
model_ids = ['gpt-3.5-turbo-16k', 'gpt-3.5-turbo-0613']
model_name = model_ids[1]
GPTllm = GPT_Turbo(model=model_name,
api_key=st.secrets['secrets']['OPENAI_API_KEY'])
openai_prompt = """
{your_task}\n
```
\n\n
Question: {question}\n
{final_instruction}
Answer:
"""
prompt = openai_prompt.format(**prompt_fields)
try:
resp = GPTllm.get_chat_completion(prompt=openai_prompt,
system_message=prompt_fields['you_are'],
temperature=0.01,
max_tokens=1500, # it's a question...
show_response=True,
stream=False)
return {'rewritten_question': resp.choices[0].delta.content,
'changed': True, 'status': 'success'}
except Exception:
return {'rewritten_question': query, 'changed': False, 'status': 'not success'}
if __name__ == '__main__':
main()
# %%