added index for more files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 apple_amazon_intel_db/* filter=lfs diff=lfs merge=lfs -text
+db_index/* filter=lfs diff=lfs merge=lfs -text
+db_index filter=lfs diff=lfs merge=lfs -text

bot.py

@@ -11,13 +11,15 @@ from langchain.memory import ConversationBufferWindowMemory
 # ConversationalRetrievalChain()
 
 
-persist_direcory = "apple_amazon_intel_db"
+persist_direcory = "db_index"
 # persist_directory = "db"
 
 embeddings = OpenAIEmbeddings()
 # db = FAISS.load_local(persist_directory, embeddings)
 chroma = Chroma(embedding_function=embeddings, persist_directory=persist_direcory)
-retriever = chroma.as_retriever(search_type="mmr", search_kwargs={"k": 10})
+# retriever = chroma.as_retriever(search_type="mmr", search_kwargs={"k": 10})
+retriever = chroma.as_retriever(search_kwargs={"k": 10})
+
 
 memory = ConversationBufferWindowMemory(
     memory_key="chat_history", return_messages=False

db_index/chroma-collections.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a93b6f7695acd8c6266ae93fd7006576543ae90070656b3f14f139fe9ac1d96
+size 557

db_index/chroma-embeddings.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee9c8a25473ff3aeee5af2cca3ef440f0b80c6a3f4fb605928c61bedd1ffae71
+size 3671568

db_index/index/id_to_uuid_c058f88c-340a-4016-92e2-428db1eee8d3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bfd2bd74cb66b0f5bc62c6f2a0f082782d9cdc4cb04feb89f122e4c8e945352
+size 5824

db_index/index/index_c058f88c-340a-4016-92e2-428db1eee8d3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54ee98ef0848ead7a6e67b4cbe425c3fd7556eeab854d1040e0d333cdf16a4ec
+size 1158108

db_index/index/index_metadata_c058f88c-340a-4016-92e2-428db1eee8d3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1476eae771b3d8b048407afe33482c83b9c0ac90067139e9a45f9cde061e4a5
+size 103

db_index/index/uuid_to_id_c058f88c-340a-4016-92e2-428db1eee8d3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e243f6aab9a67f825380ca1d8bd14685c4d6f09d2e1a75ec8c6884b06903a72d
+size 6824

index.py

@@ -0,0 +1,41 @@
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import DirectoryLoader, UnstructuredPDFLoader
+import tiktoken
+
+
+loader = DirectoryLoader(
+    "./apple_amazon_intel", glob="**/*.pdf", loader_cls=UnstructuredPDFLoader
+)
+documents = loader.load()
+
+# loader = DirectoryLoader("./data/", glob="**/*.pdf", loader_cls=PyPDFLoader)
+# documents = loader.load()
+# print(documents)
+
+
+def tiktoken_len(text):
+    tokenizer = tiktoken.encoding_for_model("gpt-4")
+    tokens = tokenizer.encode(text, disallowed_special=())
+    return len(tokens)
+
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=4000,
+    chunk_overlap=400,
+    length_function=tiktoken_len,
+    separators=["\n\n", "\n", " ", ""],
+)
+texts = text_splitter.split_documents(documents)
+
+persist_direcory = "db_index"
+
+# embeddings = OpenAIEmbeddings()
+embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+db = Chroma.from_documents(
+    texts, embedding=embeddings, persist_directory=persist_direcory
+)
+db.persist()
+print("done")