Openai activeloop data (#37)

* adding openai and activeloop data

* fixing issues with names

* concurrency

* black

* black

* revert to gradio3.50 for concurrency

---------

Co-authored-by: Omar Solano <omar@designstripe.com>

.gitignore

@@ -3,3 +3,6 @@
 deeplake_store/
 .DS_Store
 __pycache__/
+.env
+env/
+.vscode/

app.py

@@ -34,6 +34,8 @@ AVAILABLE_SOURCES_UI = [
     "Wikipedia",
     "Gen AI 360: LangChain",
     "Gen AI 360: LLMs",
+    "Activeloop",
+    "Open AI",
 ]
 
 AVAILABLE_SOURCES = [
@@ -42,6 +44,8 @@ AVAILABLE_SOURCES = [
     "wikipedia",
     "langchain_course",
     "llm_course",
+    "activeloop",
+    "openai",
 ]
 
 

cfg.py

@@ -23,7 +23,7 @@ ACTIVELOOP_TOKEN = os.getenv("ACTIVELOOP_TOKEN")
 if ACTIVELOOP_TOKEN is None:
     logger.warning("No activeloop token found, you will not be able to fetch data.")
 
-DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "dev_vector_store")
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset")
 DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
 
 # if you want to use a local dataset, set the env. variable, it overrides all others
@@ -85,7 +85,7 @@ Q:
         "embedding_model": "text-embedding-ada-002",
         "exec_option": "compute_engine",
         "use_tql": True,
-        "deep_memory": True,
+        "deep_memory": False,
         "activeloop_token": ACTIVELOOP_TOKEN,
     },
     documents_answerer_cfg={

data/process_csvs_store.py

@@ -3,8 +3,11 @@ import time
 import os
 from buster.documents_manager import DeepLakeDocumentsManager
 from deeplake.core.vectorstore import VectorStore
+from langchain.embeddings.openai import OpenAIEmbeddings
 
-DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "dev_vector_store")
+# from openai import OpenAI
+
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset")
 DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
 
 df1 = pd.read_csv("./data/llm_course.csv")
@@ -12,7 +15,10 @@ df2 = pd.read_csv("./data/hf_transformers.csv")
 df3 = pd.read_csv("./data/langchain_course.csv")
 df4 = pd.read_csv("./data/filtered_tai_v2.csv")
 df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
+df6 = pd.read_csv("./data/openai.csv")
+df7 = pd.read_csv("./data/activeloop.csv")
 
+print(len(df1), len(df2), len(df3), len(df4), len(df5), len(df6), len(df7))
 
 dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
 
@@ -27,7 +33,8 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_filename="embeddings.csv",
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
 
@@ -36,7 +43,8 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_filename="embeddings.csv",
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
 
@@ -45,7 +53,8 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_filename="embeddings.csv",
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
 
@@ -54,7 +63,8 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_filename="embeddings.csv",
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
 
@@ -63,6 +73,60 @@ dm.batch_add(
     batch_size=3000,
     min_time_interval=60,
     num_workers=32,
-    csv_filename="embeddings.csv",
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
+    csv_overwrite=False,
+)
+
+dm.batch_add(
+    df=df6,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_embeddings_filename="embeddings.csv",
+    csv_overwrite=False,
+    csv_errors_filename="tmp.csv",
+)
+
+dm.batch_add(
+    df=df7,
+    batch_size=3000,
+    min_time_interval=60,
+    num_workers=32,
+    csv_embeddings_filename="embeddings.csv",
+    csv_errors_filename="tmp.csv",
     csv_overwrite=False,
 )
+
+
+# client = OpenAI()
+
+# openai_embeddings = OpenAIEmbeddings()
+# def get_embedding(text, model="text-embedding-ada-002"):
+#     # Call to OpenAI's API to create the embedding
+#     response = client.embeddings.create(input=[text], model=model)
+
+#     # Extract the embedding data from the response
+#     embedding = response.data[0].embedding
+
+#     # Convert the ndarray to a list
+#     if isinstance(embedding, np.ndarray):
+#         embedding = embedding.tolist()
+
+#     return embedding
+
+
+# vs = VectorStore(
+#     dataset_path,
+#     runtime='compute_engine',
+#     token=os.environ['ACTIVELOOP_TOKEN']
+# )
+
+# data = vs.search(query = "select * where shape(embedding)[0] == 0")
+
+# vs.update_embedding(embedding_source_tensor = "text",
+#           query = "select * where shape(embedding)[0] == 0",
+#           exec_option = "compute_engine",
+#           embedding_function=get_embedding)
+
+# data2 = vs.search(query = "select * where shape(embedding)[0] == 0")

data/tmp.py

@@ -0,0 +1,21 @@
+# import pandas as pd
+
+# # Load the CSV
+# df = pd.read_csv('data/wiki.csv')
+
+
+# # Count the number of unique titles in the 'title' column
+# unique_titles_count = df['title']
+
+# print(len(df))
+
+# # # Remove the 'ranking' column
+# # df.drop('ranking', axis=1, inplace=True)
+
+# # # Save the CSV again
+# # df.to_csv('data/wiki.csv', index=False)
+
+
+import gradio as gr
+
+gr.themes.builder()

requirements.txt

@@ -1,3 +1,3 @@
-git+https://github.com/jerpint/buster@multiple-sources
-gradio
+git+https://github.com/jerpint/buster@better-fallback
+gradio==3.50.2
 deeplake