Advanced rag course update (#44)

* Openai activeloop data (#37)

* adding openai and activeloop data

* fixing issues with names

* concurrency

* black

* black

* revert to gradio3.50 for concurrency

---------

Co-authored-by: Omar Solano <omar@designstripe.com>

* ensure gradio version for HF

* Updates to files

* Push to advanced rag course

* Formatting

* formatting

---------

Co-authored-by: Omar Solano <omar@designstripe.com>

.gitignore

@@ -162,7 +162,9 @@ cython_debug/
 *.zip
 deeplake_store/
 .DS_Store
-
+__pycache__/
+.env
+env/
 .vscode/
 evals/
 local_dataset/

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: App",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/data/process_csvs_store.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "python": "/Users/louis/miniconda3/envs/buster/bin/python",
+            "env": {
+                "ACTIVELOOP_TOKEN": "eyJhbGciOiJIUzUxMiIsImlhdCI6MTY5Njc4MjIyMiwiZXhwIjoxNzI4NDA0NTk1fQ.eyJpZCI6Im9tYXJzb2xhbm8ifQ.BlvUc350-boJv4hnN67ksMgGSy7x4nAWcBO7R5RZ22Cw0ifR2AOmmM-RJutBWIspQDiegs03rJxXCCfprc6O_A",
+                "OPENAI_API_KEY": "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"
+            }
+        }
+    ]
+}

app.py

@@ -23,6 +23,7 @@ CONCURRENCY_COUNT = int(os.getenv("CONCURRENCY_COUNT", 64))
 
 AVAILABLE_SOURCES_UI = [
     "Gen AI 360: LLMs",
+    "Gen AU 360: Advanced RAG",
     "Gen AI 360: LangChain",
     "Towards AI Blog",
     "Activeloop Docs",
@@ -35,6 +36,7 @@ AVAILABLE_SOURCES_UI = [
 AVAILABLE_SOURCES = [
     "llm_course",
     "langchain_course",
+    "advanced_rag_course",
     "towards_ai",
     "activeloop",
     "hf_transformers",

data/markdown_parser.py

@@ -50,12 +50,13 @@ def get_title_link_from_md_title(md_title: str, title_link_data: dict):
             return data["title"], data["link"]
     # default back to course link if not found...
     print("\nNot found: ", md_title)
-    return md_title, "https://learn.activeloop.ai/courses/llms/"
+    return md_title, "https://learn.activeloop.ai/courses/rag/"
 
 
 if __name__ == "__main__":
-    folder_path = "/path/to/folder/with/md_content/"
-    folder_path = "/Users/louis/Downloads/llm_course"
+    # folder_path = "/path/to/folder/with/md_content/"
+    # This path is the link to the course folder with all md files
+    folder_path = "/Users/louis/Downloads/rag_course_advanced"
     # folder_path = "/Users/louis/Downloads/d22d1e98-345f-490d-870e-3b082938741c_Export-0a33c13f-6d42-4a94-8f23-7459e7b2c024/LangChain & Vector Databases in Production 92657e0d65da4201bfdd6db915a4eb9f"
     md_files = find_md_files(folder_path)
 
@@ -76,8 +77,8 @@ if __name__ == "__main__":
 
     # with open("data/title_link_langchaincourse.json", "r") as f:
     #     title_link_data = json.load(f)
-
-    with open("/Users/louis/Downloads/output2.json", "r") as f:
+    # This file contains a json with only two column, "title, link", to fit the title of the md files and link on the course platform.
+    with open("/Users/louis/Downloads/output.json", "r") as f:
         title_link_data = json.load(f)
 
     for md_file in tqdm(md_files):
@@ -102,7 +103,7 @@ if __name__ == "__main__":
                 chunk = {
                     "title": title,
                     "content": headers + "\n" + substring,
-                    "source": "llm_course",
+                    "source": "advanced_rag_course",
                     "url": link,
                 }
                 chunks.append(chunk)
@@ -112,4 +113,4 @@ if __name__ == "__main__":
     df = drop_outlier_chunks(df, max_tokens_by_chunk=2000)
 
     print(f"Exported {len(df)} chunks from {len(md_files)} articles.")
-    df.to_csv("llm_course.csv")
+    df.to_csv("advanced_rag_course.csv")

data/process_csvs_store.py

@@ -2,26 +2,35 @@ import pandas as pd
 import time
 import os
 from buster.documents_manager import DeepLakeDocumentsManager
+from deeplake.core.vectorstore import VectorStore
+from langchain.embeddings.openai import OpenAIEmbeddings
+import numpy as np
 
-DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "dataset-ai-tutor")
+# from openai import OpenAI
+
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset")
 DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
 
-df1 = pd.read_csv("./data/llm_course.csv")
-df2 = pd.read_csv("./data/hf_transformers.csv")
-df3 = pd.read_csv("./data/langchain_course.csv")
-df4 = pd.read_csv("./data/filtered_tai_v2.csv")
-df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
-df6 = pd.read_csv("./data/openai.csv")
-df7 = pd.read_csv("./data/activeloop.csv")
-df8 = pd.read_csv("./data/langchain_docs.csv")
+# df1 = pd.read_csv("./data/jobs.csv", encoding='ISO-8859-1')  # or 'latin1' or 'cp1252'
+# df2 = pd.read_csv("./data/hf_transformers.csv")
+# df3 = pd.read_csv("./data/langchain_course.csv")
+# df4 = pd.read_csv("./data/filtered_tai_v2.csv")
+# df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
+# df6 = pd.read_csv("./data/openai.csv")
+df1 = pd.read_csv("./advanced_rag_course.csv")
 
-print(len(df1), len(df2), len(df3), len(df4), len(df5), len(df6), len(df7), len(df8))
+# print(len(df1), len(df2), len(df3), len(df4), len(df5), len(df6))
+print(len(df1))
 
 dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
+# dataset_path = f"{DEEPLAKE_DATASET}"
+# because wrong name
+# df1['content'] = df1['cleaned_description']
+# print(np.sum(df1.content.isna()), len(df1) )
 
 dm = DeepLakeDocumentsManager(
     vector_store_path=dataset_path,
-    overwrite=True,
+    overwrite=False,
     required_columns=["url", "content", "source", "title"],
 )
 
@@ -30,77 +39,98 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
 
-dm.batch_add(
-    df=df2,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+# dm.batch_add(
+#     df=df2,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_errors_filename="tmp.csv",
+#     csv_overwrite=False,
+# )
 
-dm.batch_add(
-    df=df3,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+# dm.batch_add(
+#     df=df3,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_errors_filename="tmp.csv",
+#     csv_overwrite=False,
+# )
 
-dm.batch_add(
-    df=df4,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+# dm.batch_add(
+#     df=df4,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_errors_filename="tmp.csv",
+#     csv_overwrite=False,
+# )
 
-dm.batch_add(
-    df=df5,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+# dm.batch_add(
+#     df=df5,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_errors_filename="tmp.csv",
+#     csv_overwrite=False,
+# )
 
-dm.batch_add(
-    df=df6,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_overwrite=False,
-    csv_errors_filename="tmp.csv",
-)
+# dm.batch_add(
+#     df=df6,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_overwrite=False,
+#     csv_errors_filename="tmp.csv",
+# )
 
-dm.batch_add(
-    df=df7,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+# dm.batch_add(
+#     df=df7,
+#     batch_size=3000,
+#     min_time_interval=60,
+#     num_workers=32,
+#     csv_embeddings_filename="embeddings.csv",
+#     csv_errors_filename="tmp.csv",
+#     csv_overwrite=False,
+# )
 
-dm.batch_add(
-    df=df8,
-    batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
-    csv_embeddings_filename="embeddings.csv",
-    csv_errors_filename="tmp.csv",
-    csv_overwrite=False,
-)
+
+# client = OpenAI()
+
+# openai_embeddings = OpenAIEmbeddings()
+# def get_embedding(text, model="text-embedding-ada-002"):
+#     # Call to OpenAI's API to create the embedding
+#     response = client.embeddings.create(input=[text], model=model)
+
+#     # Extract the embedding data from the response
+#     embedding = response.data[0].embedding
+
+#     # Convert the ndarray to a list
+#     if isinstance(embedding, np.ndarray):
+#         embedding = embedding.tolist()
+
+#     return embedding
+
+
+# vs = VectorStore(
+#     dataset_path,
+#     runtime='compute_engine',
+#     token=os.environ['ACTIVELOOP_TOKEN']
+# )
+
+# data = vs.search(query = "select * where shape(embedding)[0] == 0")
+
+# vs.update_embedding(embedding_source_tensor = "text",
+#           query = "select * where shape(embedding)[0] == 0",
+#           exec_option = "compute_engine",
+#           embedding_function=get_embedding)
+
+# data2 = vs.search(query = "select * where shape(embedding)[0] == 0")

data/scrapper_to_csv.py

@@ -0,0 +1,82 @@
+import pandas as pd
+import re
+
+
+def parse_markdown_file(file_path):
+    entries = []
+    with open(file_path, "r", encoding="utf-8") as file:
+        current_url, current_title, current_content = "", "", ""
+        inside_page = False
+
+        for line in file:
+            if line.strip() == "--":  # Check for page separator
+                if inside_page:
+                    # Process the previous page
+                    process_content(
+                        entries, current_url, current_title, current_content
+                    )
+                    current_content = ""
+
+                inside_page = True
+                # Read URL and title
+                current_url = next(file).strip().split(" ", 1)[1]
+                current_title = (
+                    next(file).strip().split(" ", 1)[1].replace("\n", " ")
+                )  # Replace new lines in title
+                # Skip the next two lines (description and keywords)
+                next(file)
+                next(file)
+                # print(f"Detected Page: Title - {current_title}, URL - {current_url}")  # Debugging
+
+            elif inside_page:
+                current_content += line
+
+        if inside_page:
+            process_content(entries, current_url, current_title, current_content)
+
+    df = pd.DataFrame(entries)
+    return df
+
+
+def process_content(entries, url, title, content):
+    # Regular expression to match markdown headers
+    header_pattern = re.compile(r"^## (.+)$", re.MULTILINE)
+
+    # Split the content into sections based on headers
+    sections = re.split(header_pattern, content)
+    section_title = "Main"  # Default section title for content before the first header
+
+    # Initial content before the first header (if any)
+    if not sections[0].startswith("##") and sections[0].strip():
+        add_content_section(entries, title, url, "Main", sections[0])
+
+    # Process each section
+    for i in range(1, len(sections), 2):
+        section_header = sections[i].strip()
+        section_text = (
+            sections[i + 1].strip().replace("\n", " ")
+        )  # Replace new lines in content
+
+        add_content_section(entries, title, url, section_header, section_text)
+
+
+def add_content_section(entries, title, url, section_title, section_text):
+    full_section = f"{section_title}: {section_text}".replace(
+        "\n", " "
+    )  # Replace new lines in content
+    for j in range(0, len(full_section), 6000):
+        entries.append(
+            {
+                "title": title,
+                "url": url,
+                "source": "langchain",
+                "content": full_section[j : j + 6000],
+            }
+        )
+
+
+markdown_file_path = "data/langchain_scrape.md"
+df = parse_markdown_file(markdown_file_path)
+print("Final DataFrame:")
+print(df.head())  # Print the first few rows for verification
+df.to_csv("data/langchain.csv", index=False)

data/tmp.py

@@ -0,0 +1,121 @@
+import pandas as pd
+import time
+import os
+from deeplake.core.vectorstore import VectorStore
+from langchain.embeddings.openai import OpenAIEmbeddings
+import logging
+
+from buster.documents_manager import DeepLakeDocumentsManager
+from buster.llm_utils import get_openai_embedding_constructor
+
+# Set the logging level of `httpx` to WARNING or higher to suppress annoying INFO logs
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+openai_embedding_fn = get_openai_embedding_constructor(
+    client_kwargs={"max_retries": 10}
+)
+
+# from openai import OpenAI
+
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset")
+DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
+
+df1 = pd.read_csv("./data/langchain.csv")  # or 'latin1' or 'cp1252'
+df2 = pd.read_csv("./data/hf_transformers.csv")
+df3 = pd.read_csv("./data/langchain_course.csv")
+df4 = pd.read_csv("./data/filtered_tai_v2.csv")
+df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
+df6 = pd.read_csv("./data/openai.csv")
+df7 = pd.read_csv("./data/activeloop.csv")
+df8 = pd.read_csv("./data/llm_course.csv")
+
+print(
+    f"Number of samples: {len(df1)},{len(df2)},{len(df3)},{len(df4)},{len(df5)},{len(df6)},{len(df7)},{len(df8)}"
+)
+
+dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
+
+dm = DeepLakeDocumentsManager(
+    vector_store_path=dataset_path,
+    overwrite=True,
+)
+
+dm.batch_add(
+    df=df1,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df2,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df3,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df4,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df5,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df6,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df7,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df8,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    embedding_fn=openai_embedding_fn,
+    csv_filename="embeddings.csv",
+    csv_overwrite=False,
+)

requirements.txt

@@ -1,3 +1,3 @@
 git+https://github.com/jerpint/buster@better-fallback
 gradio==3.50.2
-deeplake==3.8.9
+deeplake==3.8.9