Improve search results with tokenizer length function and by removing TOC from the pdf.

app.py

@@ -6,27 +6,30 @@ from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.llms import HuggingFacePipeline
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import FAISS
+from transformers import GPT2TokenizerFast
 
 
 # Number of search results to query from the vector database.
-SIMILARITY_SEARCH_COUNT = 3
+SIMILARITY_SEARCH_COUNT = 8
 
-# Size of each document chunk in number of characters.
-CHUNK_SIZE = 800
+# Size of each document chunk in number of tokens.
+CHUNK_SIZE = 100
 
-# Chunk overlap in number of characters.
-CHUNK_OVERLAP = 50
+# Chunk overlap in number of tokens.
+CHUNK_OVERLAP = 10
 
 # Maximum number of output tokens.
 MODEL_MAX_LENGTH = 500
 
 
 print("Loading documents")
-loader = PyMuPDFLoader("rdna3-shader-instruction-set-architecture-feb-2023_0.pdf")
+loader = PyMuPDFLoader("rdna3.pdf")
 documents = loader.load()
 
 print("Creating chunks")
-splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)
+tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
+splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
+    tokenizer, chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP)
 chunks = splitter.split_documents(documents)
 
 print("Creating database")

rdna3-shader-instruction-set-architecture-feb-2023_0.pdf → rdna3.pdf

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bff84b0bc818446356e73ba894149b8c810549fa240a9872b46179f412fcd13b
-size 3246429
+oid sha256:81348dff61fc7b4912d86102a709a5a620819ee6d3e773a1da7a8d4c433fa45c
+size 3053930

requirements.txt

@@ -1,6 +1,7 @@
 langchain==0.0.162
 faiss-cpu==1.7.4
 huggingface-hub==0.14.1
-sentence-transformers==2.2.2
 protobuf==3.20.1
 pymupdf==1.22.2
+sentence-transformers==2.2.2
+transformers==4.28.1