feat: Split wikidocs into chunks and add to combined documents

app.py

@@ -86,6 +86,13 @@ for dirpath, _, filenames in os.walk(repo_root_dir):
 print(f".ipynb 파일의 개수: {len(ipynb_documents)}")
 
 
+## wikidocs
+import pandas as pd
+
+df = pd.read_parquet("wikidocs_14314.parquet")
+wiki_documents = text_splitter.split_documents(loader.load())
+
+
 # Split documents into chunks
 def split_documents(documents, language, chunk_size=2000, chunk_overlap=200):
     splitter = RecursiveCharacterTextSplitter.from_language(
@@ -97,12 +104,14 @@ def split_documents(documents, language, chunk_size=2000, chunk_overlap=200):
 py_docs = split_documents(py_documents, Language.PYTHON)
 mdx_docs = split_documents(mdx_documents, Language.MARKDOWN)
 ipynb_docs = split_documents(ipynb_documents, Language.PYTHON)
+wiki_docs = split_documents(wiki_documents, Language.MARKDOWN)
 
-print(f"분할된 .py 파일의 개수: {len(py_docs)}")
-print(f"분할된 .mdx 파일의 개수: {len(mdx_docs)}")
-print(f"분할된 .ipynb 파일의 개수: {len(ipynb_docs)}")
+print(f"분할된 .py 문서의 개수: {len(py_docs)}")
+print(f"분할된 .mdx 문서의 개수: {len(mdx_docs)}")
+print(f"분할된 .ipynb 문서의 개수: {len(ipynb_docs)}")
+print(f"분할된 .(wiki 문서의 개수: {len(wiki_docs)}")
 
-combined_documents = py_docs + mdx_docs + ipynb_docs
+combined_documents = py_docs + mdx_docs + ipynb_docs + wiki_docs
 print(f"총 도큐먼트 개수: {len(combined_documents)}")
 
 
@@ -132,19 +141,21 @@ cached_embeddings = CacheBackedEmbeddings.from_bytes_store(
 
 # Create and save FAISS index
 FAISS_DB_INDEX = "./langchain_faiss"
-# faiss_db = FAISS.from_documents(
-#     documents=combined_documents,
-#     embedding=cached_embeddings,
-# )
-# faiss_db.save_local(folder_path=FAISS_DB_INDEX)
+if not os.path.exists(FAISS_DB_INDEX):
+    faiss_db = FAISS.from_documents(
+        documents=combined_documents,
+        embedding=cached_embeddings,
+    )
+    faiss_db.save_local(folder_path=FAISS_DB_INDEX)
 
 # Create and save Chroma index
 CHROMA_DB_INDEX = "./langchain_chroma"
-# chroma_db = Chroma.from_documents(
-#     documents=combined_documents,
-#     embedding=cached_embeddings,
-#     persist_directory=CHROMA_DB_INDEX,
-# )
+if not os.path.exists(CHROMA_DB_INDEX):
+    chroma_db = Chroma.from_documents(
+        documents=combined_documents,
+        embedding=cached_embeddings,
+        persist_directory=CHROMA_DB_INDEX,
+    )
 
 # load vectorstore
 faiss_db = FAISS.load_local(