Add large data context index

.gitattributes

@@ -33,4 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-*json filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -1,13 +1,19 @@
 from pathlib import Path
+import os
+import openai
+openai.api_key = os.getenv("OAI_KEY")
+from llama_index.llms.openai import OpenAI
+from llama_index.embeddings.openai import OpenAIEmbedding
+import nest_asyncio
+
+nest_asyncio.apply()
+
+
 
 from llama_index.core import(SimpleDirectoryReader,
                             VectorStoreIndex, StorageContext,
                             Settings,set_global_tokenizer)
-from llama_index.llms.llama_cpp import LlamaCPP
-from llama_index.llms.llama_cpp.llama_utils import (
-    messages_to_prompt,
-    completion_to_prompt,
-)
+
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from transformers import AutoTokenizer, BitsAndBytesConfig
 from llama_index.llms.huggingface import HuggingFaceLLM
@@ -16,17 +22,15 @@ import logging
 import sys
 import streamlit as st
 from llama_index.core import load_index_from_storage
-default_bnb_config = BitsAndBytesConfig(
-                                                load_in_4bit=True,
-                                                bnb_4bit_quant_type='nf4',
-                                                bnb_4bit_use_double_quant=True,
-                                                bnb_4bit_compute_dtype=torch.bfloat16
-                                            )
+
+
+Settings.llm = OpenAI(model="gpt-3.5-turbo-instruct", temperature=0.2)
+Settings.embed_model = OpenAIEmbedding(
+    model="text-embedding-3-large", embed_batch_size=100
+)
+
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
-set_global_tokenizer(
-    AutoTokenizer.from_pretrained("NousResearch/Llama-2-13b-chat-hf").encode
-)
 
 
 def getDocs(doc_path="./data/"):
@@ -34,46 +38,29 @@ def getDocs(doc_path="./data/"):
     return documents
 
 
-def getVectorIndex(docs):
+def getVectorIndex():
     Settings.chunk_size = 512
     index_set = {}
-
-    storage_context = StorageContext.from_defaults()
-    cur_index = VectorStoreIndex.from_documents(docs, embed_model = getEmbedModel(), storage_context=storage_context)
-    storage_context.persist(persist_dir=f"./storage/book_data")
+    if os.path.isdir(f"./storage/open_ai_embedding_data_large"):
+        print("Index already exists")
+        storage_context = StorageContext.from_defaults(
+        persist_dir=f"./storage/open_ai_embedding_data_large"
+        )
+        cur_index = load_index_from_storage(
+            storage_context,
+        )
+    else:
+        print("Index does not exist, creating new index")
+        docs = getDocs()
+        storage_context = StorageContext.from_defaults()
+        cur_index = VectorStoreIndex.from_documents(docs, storage_context=storage_context)
+        storage_context.persist(persist_dir=f"./storage/open_ai_embedding_data_large")
     return cur_index
 
-
-def getLLM():
-
-    model_path = "NousResearch/Llama-2-13b-chat-hf"
-    # model_path = "NousResearch/Llama-2-7b-chat-hf"
-
-    llm = HuggingFaceLLM(
-    context_window=3900,
-    max_new_tokens=256,
-    # generate_kwargs={"temperature": 0.25, "do_sample": False},
-    tokenizer_name=model_path,
-    model_name=model_path,
-    device_map=0,
-    tokenizer_kwargs={"max_length": 2048},
-    # uncomment this if using CUDA to reduce memory usage
-    model_kwargs={"torch_dtype": torch.float16,
-    "quantization_config": default_bnb_config,
-    }
-    )
-    return llm
-
-
 def getQueryEngine(index):
-    query_engine = index.as_chat_engine(llm=getLLM())
+    query_engine = index.as_chat_engine()
     return query_engine
 
-def getEmbedModel():
-    embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en-v1.5")
-    return embed_model
-
-
 
 
 
@@ -94,14 +81,17 @@ if "messages" not in st.session_state.keys(): # Initialize the chat messages his
 
 @st.cache_resource(show_spinner=False)
 def load_data():
-    index = getVectorIndex(getDocs())
+    index = getVectorIndex()
     return index
-    query_engine = getQueryEngine(index)
-
+import time
+s_time = time.time()
 index = load_data()
+e_time = time.time()
+
+print(f"It took {e_time - s_time} to load index")
 
 if "chat_engine" not in st.session_state.keys(): # Initialize the chat engine
-        st.session_state.chat_engine = index.as_chat_engine(llm=getLLM(),chat_mode="condense_question", verbose=True)
+        st.session_state.chat_engine = index.as_chat_engine(chat_mode="condense_plus_context", verbose=True)
 
 if prompt := st.chat_input("Your question"): # Prompt for user input and save to chat history
     st.session_state.messages.append({"role": "user", "content": prompt})

storage/.gitattributes

@@ -0,0 +1 @@
+*json filter=lfs diff=lfs merge=lfs -text

storage/open_ai_embedding_data/default__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87956943cb8e0633d9df6a98d31a12c9528901114a79b39c179734999cee7163
+size 244449202

storage/open_ai_embedding_data/docstore.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a7d9222ba808d2bf326098e84b7b959ba0104c923ce7f87c782ecfd93404325
+size 29962555

storage/open_ai_embedding_data/graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0a77744010862225c69da83c585f4f8a42fd551b044ce530dbb1eb6e16742c
+size 18

storage/open_ai_embedding_data/image__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17ed74c1649a438e518a8dc56a7772913dfe1ea7a7605bce069c63872431455
+size 72

storage/open_ai_embedding_data/index_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd52cfd6aba4fb5c0774b7e3d38ddcda21e0cf5344a86c8eaf7c8690bb451bcd
+size 589927

storage/open_ai_embedding_data_large/default__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ad49abac9bff5c529bb2446b985e7ae14a74328d6d2293f6d421326b3851538
+size 487945734

storage/open_ai_embedding_data_large/docstore.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3983b645d0aefc3d92158d573cb9bc4d3f79077a77066e140d8e725dd7e085b5
+size 29962555

storage/open_ai_embedding_data_large/graph_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0a77744010862225c69da83c585f4f8a42fd551b044ce530dbb1eb6e16742c
+size 18

storage/open_ai_embedding_data_large/image__vector_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17ed74c1649a438e518a8dc56a7772913dfe1ea7a7605bce069c63872431455
+size 72

storage/open_ai_embedding_data_large/index_store.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a7b6dad9b8f418dfd26132e54203b8dca1374dc8e8c3199d5e9d001816f3cf
+size 589927