first commit

.gitignore

@@ -0,0 +1,21 @@
+# env
+.env*
+healthhack-store-firebase-adminsdk-sq7ne-32400d49de.json
+/venv
+
+# notebooks
+*.ipynb
+/.ipynb_checkpoints
+
+# docs
+/docs
+docs.zip
+
+# indexes
+indexes/
+
+# firebase
+.firebase
+
+# others
+*.log

db_firestore.py

@@ -0,0 +1,78 @@
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings.huggingface import HuggingFaceBgeEmbeddings
+import firebase_admin
+from firebase_admin import credentials, storage
+import json, os, dotenv
+from dotenv import load_dotenv
+load_dotenv()
+
+os.environ["FIREBASE_CREDENTIAL"] = dotenv.get_key(dotenv.find_dotenv(), "FIREBASE_CREDENTIAL")
+cred = credentials.Certificate(json.loads(os.environ.get("FIREBASE_CREDENTIAL")))
+firebase_admin.initialize_app(cred,{'storageBucket': 'healthhack-store.appspot.com'}) # connecting to firebase
+
+
+def get_store(index_name, embeddings = None):
+    while index_name[-1]=="/":
+        index_name = index_name[:-1]
+    dir = index_name.split("/")
+    
+    ## Check if path exists locally
+    for i in range(len(dir)):
+        path = '/'.join(dir[:i+1])
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+    ## Check if file exists locally, get from blob
+    if (not os.path.exists(index_name+"/index.faiss") or
+        not os.path.exists(index_name+"/index.pkl")
+        ):
+        bucket = storage.bucket()
+        blob = bucket.blob(f"{index_name}/index.pkl")
+        blob.download_to_filename(f"{index_name}/index.pkl")
+        bucket = storage.bucket()
+        blob = bucket.blob(f"{index_name}/index.faiss")
+        blob.download_to_filename(f"{index_name}/index.faiss")
+    
+    ## check embeddings, default to BGE
+    if embeddings is None:
+        model_name = "bge-large-en-v1.5"
+        model_kwargs = {"device": "cpu"}
+        encode_kwargs = {"normalize_embeddings": True}
+        embeddings = HuggingFaceBgeEmbeddings(
+            # model_name=model_name, 
+            model_kwargs = model_kwargs,
+            encode_kwargs = encode_kwargs)
+
+    ## load store from local
+    store = FAISS.load_local(index_name, embeddings)
+    return store
+
+def update_store_from_local(index_name):
+    while index_name[-1]=="/":
+        index_name = index_name[:-1]
+    pathdir = index_name.split("/")
+    
+    ## Check if path exists locally
+    for i in range(len(pathdir)):
+        path = '/'.join(pathdir[:i+1])
+        if not os.path.exists(path):
+            raise Exception("Index name does not exist locally")
+
+    ## Check if file exists locally, get from blob
+    if (not os.path.exists(index_name+"/index.faiss") or
+        not os.path.exists(index_name+"/index.pkl")
+        ):
+        raise("Index is missing some files (index.faiss, index.pkl)")
+    
+    ## Update store
+    bucket = storage.bucket()
+    blob = bucket.blob(index_name+"/index.faiss")
+    blob.upload_from_filename(index_name+"/index.faiss")
+    blob = bucket.blob(index_name+"/index.pkl")
+    blob.upload_from_filename(index_name+"/index.pkl")
+    return True
+    
+    
+
+if __name__ == "__main__":
+    print("y r u running dis")

requirements.txt

@@ -0,0 +1,17 @@
+transformers
+langchain==0.0.352
+langchain-openai
+sentence_transformers
+python-dotenv
+numexpr
+ipywidgets
+ipykernel
+pypdf
+aspose-words
+tiktoken
+faiss-cpu
+streamlit
+firebase-admin
+
+--index-url https://download.pytorch.org/whl/cu113
+torch==2.1.2

streamlit/app.py

@@ -0,0 +1,60 @@
+import streamlit as st
+from langchain_community.document_loaders import TextLoader
+from langchain_openai import AzureOpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain.docstore.document import Document
+
+import openai
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+import tiktoken
+
+import os
+
+from dotenv import load_dotenv
+load_dotenv()
+if not os.environ.get("OPENAI_API_KEY"):
+    raise Exception("No OpenAI Key detected")
+
+embeddings = OpenAIEmbeddings(deployment="textembedding", chunk_size = 16, api_key = os.environ["OPENAI_API_KEY"])
+index_name = "SCLC"
+store = FAISS.load_local(index_name, embeddings)
+llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+
+from langchain.prompts.few_shot import FewShotPromptTemplate
+from langchain.prompts.prompt import PromptTemplate
+from operator import itemgetter
+from langchain.schema import StrOutputParser
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+
+TEMPLATE = """You are a chatbot.
+Here is the context:
+{context}
+----------------------------------------------------------------
+You are to reply the following question, with reference to the above context.
+Question:
+{question}
+----------------------------------------------------------------
+Your reply:
+"""
+
+prompt = PromptTemplate(
+    input_variables = ["question", "context"],
+    template = TEMPLATE
+)
+retriever = store.as_retriever(search_type="similarity", search_kwargs={"k":2})
+def format_docs(docs):
+    return "\n--------------------\n".join(doc.page_content for doc in docs)
+
+chain = ({"context": retriever | format_docs, "question": RunnablePassthrough()} | 
+    prompt | 
+    llm | 
+    StrOutputParser()
+)
+
+
+st.title("test")
+
+t = st.text_input("Input")
+st.write(chain.invoke(t))