16k prompt (#19)

* 16k context window

* threshold and top_k changed to 10

* changed max_tokens to fit new context window

* changed before and after prompts with more details

* added number of chunks per source

* formatting

* used online dataset

* remove before docs, remove summarize keywords from prompt

* answer between 1 and 8 paragraphs

* add prompt text_before_docs back + overall prompt modifications

* format with black

* expand chatbot box to full height and change demo theme

* prompt change, ask to return 5 paragraphs

* change space name to dev

* new dataset curated TAI, no wiki

* new dataset curated TAI, no wiki

* change sources

* adding all datasets

* added all new csvs

* rm tmp

* update vector db push

* fix prompt for OOD detector

* Added cleaner sources and sources in refs

* ran black formatting

---------

Co-authored-by: Omar Solano <omar@designstripe.com>
Co-authored-by: Omar Solano <omarsolano27@gmail.com>
Co-authored-by: Jeremy Pinto <jerpint@gmail.com>

README.md

@@ -1,8 +1,8 @@
 ---
-title: TowardsAI 🤝 Buster
+title: TowardsAI 🤝 Buster - Dev
 emoji: 🤖
-colorFrom: pink
-colorTo: green
+colorFrom: blue
+colorTo: blue
 sdk: gradio
 app_file: app.py
 pinned: false

app.py

@@ -18,7 +18,21 @@ logging.getLogger("httpx").setLevel(logging.WARNING)
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-AVAILABLE_SOURCES = ["towardsai", "wikipedia", "langchain_course"]
+AVAILABLE_SOURCES_UI = [
+    "Toward's AI",
+    "HuggingFace",
+    "Wikipedia",
+    "Gen AI 360: LangChain",
+    "Gen AI 360: LLMs",
+]
+
+AVAILABLE_SOURCES = [
+    "towards_ai",
+    "hf_transformers",
+    "wikipedia",
+    "langchain_course",
+    "llm_course",
+]
 
 
 def format_sources(matched_documents: pd.DataFrame) -> str:
@@ -26,17 +40,29 @@ def format_sources(matched_documents: pd.DataFrame) -> str:
         return ""
 
     documents_answer_template: str = "📝 Here are the sources I used to answer your question:\n\n{documents}\n\n{footnote}"
-    document_template: str = "[🔗 {document.title}]({document.url}), relevance: {document.similarity_to_answer:2.1f} %"
+    document_template: str = "[🔗 {document.source}: {document.title}]({document.url}), highest relevance: {document.similarity_to_answer:2.1f} % | # total chunks matched: {document.repetition:d}"
 
     matched_documents.similarity_to_answer = (
         matched_documents.similarity_to_answer * 100
     )
 
+    matched_documents["repetition"] = matched_documents.groupby("title")[
+        "title"
+    ].transform("size")
+
     # drop duplicates, keep highest ranking ones
     matched_documents = matched_documents.sort_values(
         "similarity_to_answer", ascending=False
     ).drop_duplicates("title", keep="first")
 
+    # Revert back to correct display
+    display_source_to_ui = {
+        ui: src for ui, src in zip(AVAILABLE_SOURCES, AVAILABLE_SOURCES_UI)
+    }
+    matched_documents["source"] = matched_documents["source"].replace(
+        display_source_to_ui
+    )
+
     documents = "\n".join(
         [
             document_template.format(document=document)
@@ -77,7 +103,13 @@ def get_answer(history, sources: Optional[list[str]] = None):
         completion = get_empty_source_completion(user_input)
 
     else:
-        completion = buster.process_input(user_input, sources=sources)
+        # Go to code names
+        display_ui_to_source = {
+            ui: src for ui, src in zip(AVAILABLE_SOURCES_UI, AVAILABLE_SOURCES)
+        }
+
+        sources_renamed = [display_ui_to_source[disp] for disp in sources]
+        completion = buster.process_input(user_input, sources=sources_renamed)
 
     history[-1][1] = ""
 
@@ -87,26 +119,32 @@ def get_answer(history, sources: Optional[list[str]] = None):
         yield history, completion
 
 
-demo = gr.Blocks()
+CSS = """
+.contain { display: flex; flex-direction: column; }
+.gradio-container { height: 100vh !important; }
+#component-0 { height: 100%; }
+#chatbot { flex-grow: 1; overflow: auto;}
+"""
+theme = gr.themes.Base()
+demo = gr.Blocks(css=CSS, theme=theme)
 with demo:
     with gr.Row():
         gr.Markdown(
-            "<h3><center>Buster 🤖: A Question-Answering Bot for your documentation</center></h3>"
+            "<h3><center>Toward's AI x Buster 🤖: A Question-Answering Bot for anything AI-related</center></h3>"
         )
-
     source_selection = gr.Dropdown(
-        choices=AVAILABLE_SOURCES,
+        choices=AVAILABLE_SOURCES_UI,
         label="Select Sources",
-        value=AVAILABLE_SOURCES,
+        value=AVAILABLE_SOURCES_UI,
         multiselect=True,
     )
 
-    chatbot = gr.Chatbot()
+    chatbot = gr.Chatbot(elem_id="chatbot")
 
     with gr.Row():
         question = gr.Textbox(
             label="What's your question?",
-            placeholder="Ask a question to AI stackoverflow here...",
+            placeholder="Ask a question to our AI tutor here...",
             lines=1,
         )
         submit = gr.Button(value="Send", variant="secondary")

cfg.py

@@ -43,9 +43,11 @@ buster_cfg = BusterConfig(
         "use_reranking": True,
         "invalid_question_response": "This question does not seem relevant to my current knowledge.",
         "check_question_prompt": """You are a chatbot, answering questions about large language models and artificial intelligence.
-Users will ask all sorts of questions, and some might be tangentially related.
+Your job is to determine whether user's question is valid or not. Users will not always submit a question either.
+Users will ask all sorts of questions, and some might be tangentially related to artificial intelligence.
 Users will learn to build LLM-powered apps, with LangChain & Deep Lake among other technologies.
-As long as a question is somewhat related to the topic, respond 'true'. If a question is completely unrelated, respond 'false'.
+As long as a question is somewhat related to the topic of AI, respond 'true'. If a question is on a different subject or unrelated, respond 'false'.
+Make sure the question is a valid question.
 
 For example:
 
@@ -55,7 +57,8 @@ true
 Q: What is the meaning of life?
 false
 
-A user will now submit a question. Respond 'true' if it is valid, respond 'false' if it is invalid.""",
+Q:
+""",
         "completion_kwargs": {
             "model": "gpt-3.5-turbo",
             "stream": False,
@@ -64,9 +67,9 @@ A user will now submit a question. Respond 'true' if it is valid, respond 'false
     },
     retriever_cfg={
         "path": f"{DEEPLAKE_DATASET_PATH}",
-        "top_k": 3,
-        "thresh": 0.7,
-        "max_tokens": 2000,
+        "top_k": 10,
+        "thresh": 0.55,
+        "max_tokens": 13000,
         "embedding_model": "text-embedding-ada-002",
         "exec_option": "compute_engine",
         "use_tql": True,
@@ -77,41 +80,41 @@ A user will now submit a question. Respond 'true' if it is valid, respond 'false
     },
     completion_cfg={
         "completion_kwargs": {
-            "model": "gpt-3.5-turbo",
+            "model": "gpt-3.5-turbo-16k",
             "stream": True,
             "temperature": 0,
         },
     },
     tokenizer_cfg={
-        "model_name": "gpt-3.5-turbo",
+        "model_name": "gpt-3.5-turbo-16k",
     },
     documents_formatter_cfg={
-        "max_tokens": 3500,
+        "max_tokens": 13500,
         "columns": ["content", "source", "title"],
     },
     prompt_formatter_cfg={
-        "max_tokens": 3500,
+        "max_tokens": 13500,
         "text_before_docs": (
-            "You are a chatbot assistant answering users' questions about towardsAI content, a blog about applied artificial intelligence (AI)."
+            "You are a witty AI teacher, helpfully answering questions from students of an applied artificial intelligence course on Large Language Models (LLMs or llm). Topics covered include training models, fine tuning models, giving memory to LLMs, prompting, hallucinations and bias, vector databases, transformer architectures, embeddings, Langchain, making LLMs interact with tool use, AI agents, reinforcement learning with human feedback. Questions should be understood with this context."
             "You are provided information found in the json documentation. "
-            "Only respond with infomration inside the json documentation. DO NOT use additional information, even if you know the answer. "
-            "If the answer is in the documentation, summarize it in a helpful way to the user. "
+            "Only respond with information inside the json documentation. DO NOT use additional information, even if you know the answer. "
+            "If the answer is in the documentation, answer the question (depending on the questions and the variety of relevant information in the json documentation, answer in 5 paragraphs."
             "If the documentation does not discuss the topic related to the question, kindly respond that you cannot answer the question because it is not part of your knowledge. "
             "Here is the information you can use (json documentation): "
         ),
         "text_after_docs": (
             "REMEMBER:\n"
-            "You are a chatbot assistant answering users' questions about towardsAI content, a blog about applied artificial intelligence (AI)."
-            "You are provided information found in the . "
+            "You are a witty AI teacher, helpfully answering questions from students of an applied artificial intelligence course on Large Language Models (LLMs or llm). Topics covered include training models, fine tuning models, giving memory to LLMs, prompting, hallucinations and bias, vector databases, transformer architectures, embeddings, Langchain, making LLMs interact with tool use, AI agents, reinforcement learning with human feedback. Questions should be understood with this context."
+            "You are provided information found in the json documentation. "
             "Here are the rules you must follow:\n"
-            "* Only respond with infomration inside the json documentation. DO NOT providew additional information, even if you know the answer. "
-            "* If the answer is in the documentation, summarize it in a helpful way to the user. "
+            "* Only respond with information inside the json documentation. DO NOT provide additional information, even if you know the answer. "
+            "* If the answer is in the documentation, answer the question (depending on the questions and the variety of relevant information in the json documentation, answer in 5 paragraphs. "
             "* If the documentation does not discuss the topic related to the question, kindly respond that you cannot answer the question because it is not part of your knowledge. "
-            "* Only summarize the information in the json documentation, do not respond otherwise. "
-            "* Do not refer to the documentation directly, but use the instructions provided within it to answer questions. "
+            "* Only use information summarized from the json documentation, do not respond otherwise. "
+            "* Do not refer to the json documentation directly, but use the instructions provided within it to answer questions. "
             "* Do not reference any links, urls or hyperlinks in your answers.\n"
             "* Make sure to format your answers in Markdown format, including code block and snippets.\n"
-            "* If you do not know the answer to a question, or if it is completely irrelevant to the library usage, simply reply with:\n"
+            "* If you do not know the answer to a question, or if it is completely irrelevant to the AI courses, simply reply with:\n"
             "'I'm sorry, but I am an AI language model trained to assist with questions related to AI. I cannot answer that question as it is not relevant to the topics I'm trained on. Is there anything else I can assist you with?'"
             "For example:\n"
             "What is the meaning of life for a qa bot?\n"
@@ -139,4 +142,5 @@ def setup_buster(buster_cfg):
     buster: Buster = Buster(
         retriever=retriever, document_answerer=document_answerer, validator=validator
     )
+
     return buster

data/embed_documents.py

@@ -2,9 +2,9 @@ import pandas as pd
 from buster.documents_manager import DeepLakeDocumentsManager
 
 if __name__ == "__main__":
-    vector_store_path = "wiki_tai_langchain"
-    chunk_file = "./data/wiki_tai_langchain.csv"
-    overwrite = True
+    vector_store_path = "/Users/louis/Downloads/wiki_tai_langchain_hf_llm"
+    chunk_file = "./llm_course.csv"
+    overwrite = False
 
     df = pd.read_csv(chunk_file)
 

data/markdown_parser.py

@@ -50,13 +50,13 @@ def get_title_link_from_md_title(md_title: str, title_link_data: dict):
             return data["title"], data["link"]
     # default back to course link if not found...
     print("\nNot found: ", md_title)
-    return md_title, "https://learn.activeloop.ai/courses/langchain/"
+    return md_title, "https://learn.activeloop.ai/courses/llms/"
 
 
 if __name__ == "__main__":
     folder_path = "/path/to/folder/with/md_content/"
-    folder_path = "/Users/jeremypinto/Downloads/d22d1e98-345f-490d-870e-3b082938741c_Export-0a33c13f-6d42-4a94-8f23-7459e7b2c024"
-
+    folder_path = "/Users/louis/Downloads/llm_course"
+    # folder_path = "/Users/louis/Downloads/d22d1e98-345f-490d-870e-3b082938741c_Export-0a33c13f-6d42-4a94-8f23-7459e7b2c024/LangChain & Vector Databases in Production 92657e0d65da4201bfdd6db915a4eb9f"
     md_files = find_md_files(folder_path)
 
     headers_to_split_on = [
@@ -74,14 +74,16 @@ if __name__ == "__main__":
 
     from tqdm import tqdm
 
-    with open("title_link_langchaincourse.json", "r") as f:
+    # with open("data/title_link_langchaincourse.json", "r") as f:
+    #     title_link_data = json.load(f)
+
+    with open("/Users/louis/Downloads/output2.json", "r") as f:
         title_link_data = json.load(f)
 
     for md_file in tqdm(md_files):
         md_title = md_file["title"]
         md_raw_content = md_file["content"]
         md_header_splits = markdown_splitter.split_text(md_raw_content)
-
         title, link = get_title_link_from_md_title(
             md_title, title_link_data=title_link_data
         )
@@ -100,7 +102,7 @@ if __name__ == "__main__":
                 chunk = {
                     "title": title,
                     "content": headers + "\n" + substring,
-                    "source": "TAI Course",
+                    "source": "llm_course",
                     "url": link,
                 }
                 chunks.append(chunk)
@@ -110,4 +112,4 @@ if __name__ == "__main__":
     df = drop_outlier_chunks(df, max_tokens_by_chunk=2000)
 
     print(f"Exported {len(df)} chunks from {len(md_files)} articles.")
-    df.to_csv("langchain_course.csv")
+    df.to_csv("llm_course.csv")

data/process_csvs_store.py

@@ -0,0 +1,68 @@
+import pandas as pd
+import time
+import os
+from buster.documents_manager import DeepLakeDocumentsManager
+from deeplake.core.vectorstore import VectorStore
+
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "dev_vector_store")
+DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
+
+df1 = pd.read_csv("./data/llm_course.csv")
+df2 = pd.read_csv("./data/hf_transformers.csv")
+df3 = pd.read_csv("./data/langchain_course.csv")
+df4 = pd.read_csv("./data/filtered_tai_v2.csv")
+df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
+
+
+dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
+
+dm = DeepLakeDocumentsManager(
+    vector_store_path=dataset_path,
+    overwrite=True,
+    required_columns=["url", "content", "source", "title"],
+)
+
+dm.batch_add(
+    df=df1,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df2,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df3,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df4,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df5,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)

data/remove_extra_notion_titles.py

@@ -0,0 +1,19 @@
+import os
+
+# Set your folder path
+folder_path = "/Users/louis/Downloads/llm_course"
+# Get all filenames from the folder
+filenames = os.listdir(folder_path)
+
+# Iterate through each filename
+for filename in filenames:
+    last_space_index = filename.rfind(" ")
+
+    # If a space is found, rename the file
+    if last_space_index != -1:
+        new_filename = filename[:last_space_index]
+        old_path = os.path.join(folder_path, filename)
+        new_path = os.path.join(folder_path, new_filename)
+
+        os.rename(old_path, new_path)
+        print(f"Renamed {filename} to {new_filename}")