Update code

ingest.py

@@ -6,7 +6,7 @@ from markdown import markdown
 import pickle
 from bs4 import BeautifulSoup
 from langchain.text_splitter import CharacterTextSplitter
-from langchain.embeddings import HuggingFaceEmbeddings, OpenAIEmbeddings
+from langchain.embeddings import HuggingFaceInstructEmbeddings, OpenAIEmbeddings
 from langchain.vectorstores import FAISS
 from InstructorEmbedding import INSTRUCTOR
 
@@ -25,22 +25,26 @@ for p in Path("docs").rglob("*"):
         continue
     if str(p).lower().endswith(('.md', '.mdx')):
         with open(p) as f:
-            print(p)
             filename = os.path.splitext(p)[0]
             docs.append(clean_data(f.read()))
-            metadatas.append({"source": filename})
+            newfile_name = filename.replace("\\", "/")[5:]
+            print("file:" + newfile_name)
+            metadatas.append({"source": newfile_name})
 
 text_splitter = CharacterTextSplitter(
     separator="\n",
-    chunk_size=512,
-    chunk_overlap=64,
+    chunk_size=768,
+    chunk_overlap=128,
     length_function=len,
 )
 
 documents = text_splitter.create_documents(docs, metadatas=metadatas)
 
 print("making embedding")
-embedding = HuggingFaceEmbeddings()
+model_name = "hkunlp/instructor-large"
+embed_instruction = "Represent the text from the Hugging Face code documentation"
+query_instruction = "Query the most relevant text from the Hugging Face code documentation"
+embedding = HuggingFaceInstructEmbeddings(model_name=model_name, embed_instruction=embed_instruction, query_instruction=query_instruction)
 
 print("beginning construction of faiss")
 search_index = FAISS.from_documents(documents, embedding)

ingest_examples.py

@@ -3,7 +3,7 @@ import os
 from pathlib import Path
 import pickle
 from langchain.vectorstores import FAISS
-from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
+from langchain.embeddings import HuggingFaceInstructEmbeddings
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.prompts.example_selector import \
     SemanticSimilarityExampleSelector
@@ -41,11 +41,16 @@ rephrase_documents = [
     }
 ]
 
+model_name = "hkunlp/instructor-large"
+embed_instruction = "Represent the text from the Hugging Face code documentation"
+query_instruction = "Query the most relevant text from the Hugging Face code documentation"
+embedding = HuggingFaceInstructEmbeddings(model_name=model_name, embed_instruction=embed_instruction, query_instruction=query_instruction)
+
 example_selector = SemanticSimilarityExampleSelector.from_examples(
     # This is the list of examples available to select from.
     rephrase_documents,
     # This is the embedding class used to produce embeddings which are used to measure semantic similarity.
-    HuggingFaceEmbeddings(),
+    embedding,
     # This is the VectorStore class that is used to store the embeddings and do a similarity search over.
     FAISS,
     # This is the number of examples to produce.