Update app.py

app.py

@@ -1,6 +1,6 @@
 import os
 from langchain_community.document_loaders import PyMuPDFLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter
 from langchain_qdrant import QdrantVectorStore
 from langchain_community.vectorstores import Qdrant
 from langchain.prompts import ChatPromptTemplate
@@ -35,6 +35,13 @@ text_splitter = RecursiveCharacterTextSplitter(
 )
 rag_documents = text_splitter.split_documents(documents)
 
+# Alternative chunking: Tokens (more accurate for OpenAI models)
+token_text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
+    encoding="cl100k_base", chunk_size=100, chunk_overlap=0
+)
+token_rag_documents = token_text_splitter.split_documents(documents)
+# TO DO ^^ test
+
 embedding = OpenAIEmbeddings(model="text-embedding-3-small")
 
 # Create the vector store