Upload 10 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chroma/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

chroma/2c377c44-f8d1-490a-87ff-86458f8adab8/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56bfe039be3845d219acd5b590595ec7caa0e9a5d4bd61aa4c9407b5781eb34
+size 25136000

chroma/2c377c44-f8d1-490a-87ff-86458f8adab8/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6086851e0ed6b004edb9fcd6d976b1aee7d2424bc955702e7373dbdb5753f3
+size 100

chroma/2c377c44-f8d1-490a-87ff-86458f8adab8/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87170a517300fef8206aeca2a7217e542e4f6f6af9aa0bda929e9eeed20515e2
+size 230019

chroma/2c377c44-f8d1-490a-87ff-86458f8adab8/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502471760e7e55abba071e374c9304b326d97d731ef4c5d91542be343a67530b
+size 16000

chroma/2c377c44-f8d1-490a-87ff-86458f8adab8/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fa1bf89e90612633fb2e23793b030e1989531339a26e8c21a2a695501ecc4d
+size 34768

chroma/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29c0d6469614d37b7eed2c39a30c5d1106fe23d895ae7b946fee6b732bf031c5
+size 46448640

compare_embeddings.py

@@ -0,0 +1,20 @@
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.evaluation import load_evaluator
+
+
+def main():
+    # Get embedding for a word.
+    embedding_function = OpenAIEmbeddings()
+    vector = embedding_function.embed_query("apple")
+    print(f"Vector for 'apple': {vector}")
+    print(f"Vector length: {len(vector)}")
+
+    # Compare vector of two words
+    evaluator = load_evaluator("pairwise_embedding_distance")
+    words = ("apple", "iphone")
+    x = evaluator.evaluate_string_pairs(prediction=words[0], prediction_b=words[1])
+    print(f"Comparing ({words[0]}, {words[1]}): {x}")
+
+
+if __name__ == "__main__":
+    main()

create_database.py

@@ -0,0 +1,60 @@
+from langchain.document_loaders import DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema import Document
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores.chroma import Chroma
+import os
+import shutil
+
+CHROMA_PATH = "chroma"
+DATA_PATH = "data/final_crawl"
+
+
+def main():
+    generate_data_store()
+
+
+def generate_data_store():
+    documents = load_documents()
+    chunks = split_text(documents)
+    save_to_chroma(chunks)
+
+
+def load_documents():
+    loader = DirectoryLoader(DATA_PATH, glob="*.md")
+    documents = loader.load()
+    return documents
+
+
+def split_text(documents: list[Document]):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=300,
+        chunk_overlap=100,
+        length_function=len,
+        add_start_index=True,
+    )
+    chunks = text_splitter.split_documents(documents)
+    print(f"Split {len(documents)} documents into {len(chunks)} chunks.")
+
+    document = chunks[10]
+    print(document.page_content)
+    print(document.metadata)
+
+    return chunks
+
+
+def save_to_chroma(chunks: list[Document]):
+    # Clear out the database first.
+    if os.path.exists(CHROMA_PATH):
+        shutil.rmtree(CHROMA_PATH)
+
+    # Create a new DB from the documents.
+    db = Chroma.from_documents(
+        chunks, OpenAIEmbeddings(), persist_directory=CHROMA_PATH
+    )
+    db.persist()
+    print(f"Saved {len(chunks)} chunks to {CHROMA_PATH}.")
+
+
+if __name__ == "__main__":
+    main()

query_data.py

@@ -0,0 +1,72 @@
+import argparse
+from langchain.vectorstores.chroma import Chroma
+from langchain.embeddings import OpenAIEmbeddings
+from llamaapi import LlamaAPI
+from langchain.prompts import ChatPromptTemplate
+
+CHROMA_PATH = "chroma"
+
+PROMPT_TEMPLATE = """
+Answer the question based only on the following context:
+
+{context}
+
+---
+
+Answer the question based on the above context: {question}
+"""
+
+def generate_reworded_question(prompt):
+    llama = LlamaAPI('LL-0tVJ5OwMLdglnL5Okd94ScFHyT6FMPP33oClu8i5cXWPScRswldmqXI7VH1JaT3x')
+
+    # API Request
+    api_request_json = {
+        "model": "llama-13b-chat",
+        "messages": [
+            {"role": "user", "content": prompt},
+        ], 
+        "max_tokens": 200,  # Set max_tokens to control the length of the generated question
+        "temperature": 0.1,  # Adjust temperature to control the creativity of the generated question
+        "top_p": 0.9  # Adjust top_p to control the diversity of the generated question
+    }
+
+    try:
+        # Run llama
+        response = llama.run(api_request_json)
+        response_json = response.json()
+        reworded_questions = [choice['message']['content'] for choice in response_json['choices']]
+        return reworded_questions
+    except Exception as e:
+        print(f"Error generating reworded questions: {e}")
+        return []  # Return an empty list if there's an error
+
+def main():
+    # Create CLI.
+    parser = argparse.ArgumentParser()
+    parser.add_argument("query_text", type=str, help="The query text.")
+    args = parser.parse_args()
+    query_text = args.query_text
+
+    # Prepare the DB.
+    embedding_function = OpenAIEmbeddings()
+    db = Chroma(persist_directory=CHROMA_PATH, embedding_function=embedding_function)
+    
+    # Search the DB.
+    results = db.similarity_search_with_relevance_scores(query_text, k=3)
+    if len(results) == 0 or results[0][1] < 0.7:
+        response_text = generate_reworded_question(query_text)
+        formatted_response = f"Response: {response_text}"
+    else:    
+        context_text = "\n\n---\n\n".join([doc.page_content for doc, _score in results])
+        prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)
+        prompt = prompt_template.format(context=context_text, question=query_text)
+        #print(prompt)
+        response_text = generate_reworded_question(prompt)
+        sources = [doc.metadata.get("source", None) for doc, _score in results]
+        formatted_response = f"\n\nResponse:\n {response_text}\n\nSources: {sources}"
+    
+    print(formatted_response)
+
+# Call the main function
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,5 @@
+langchain
+unstructured # Document loading
+chromadb # Vector storage
+openai # For embeddings
+tiktoken  # For embeddings