first commit

.gitattributes

@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/chroma-collections.parquet filter=lfs diff=lfs merge=lfs -text
+data/index/index_68425e73-3b8d-47fd-9dd0-ceb0ad113836.bin filter=lfs diff=lfs merge=lfs -text
+data/index/index_569e1be0-57b0-4ef4-abcc-a40cccd1b383.bin filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,100 @@
+import os
+
+import streamlit as st
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import TextLoader
+from langchain.text_splitter import CharacterTextSplitter
+
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+
+
+st.set_page_config(page_title="CoreMind AI", layout="wide")
+
+st.header("CoreMind AI")
+
+st.sidebar.title("Options")
+
+# Set your OpenAI API key here or delete and put it in your environment variables as OPENAI_API_KEY
+openai_key = st.sidebar.text_input("OpenAI API Key", type="password", key="openai_api_key")
+
+
+if openai_key:
+    os.environ["OPENAI_API_KEY"] = openai_key
+
+    # set up data
+
+    loader = TextLoader("raw_data.txt")
+
+    embeddings = OpenAIEmbeddings()
+
+    docsearch = Chroma(persist_directory="data", embedding_function=embeddings)
+
+
+    qa_temperature = st.sidebar.slider("QA Temperature", min_value=0.0, max_value=2.0, value=0.5, step=0.01, key="temperature")
+    qa_model = st.sidebar.selectbox("QA Model", ["gpt-3.5-turbo"], key="model")
+
+    # st.sidebar.markdown("----")
+
+    # allow the user to insert new data
+    # new_data = st.sidebar.text_area("Add new data", key="new_data")
+    # if new_data:
+    #     # save data to user_data
+    #     with open("user_data/raw_data.txt", "a") as f:
+    #         f.write(new_data)
+
+    #     loader = TextLoader("user_data/raw_data.txt")
+    #     documents = loader.load()
+    #     text_splitter = CharacterTextSplitter(chunk_size=3000, chunk_overlap=300)
+    #     texts = text_splitter.split_documents(documents)
+    #     docsearch.add_documents(texts)
+    #     docsearch.persist()
+
+
+    # st.sidebar.markdown("----")
+    # choice =  st.sidebar.selectbox("Example prompts", ["Compare the revenues and operating profits between 1996, 1995 and 1994", "Get me the common stock investments values and organise them in a format easy for plotting"])
+    # button = st.sidebar.button("Run Example Prompt")
+
+
+    qa = RetrievalQA.from_chain_type(
+        llm=OpenAI(temperature=qa_temperature, model_name=qa_model), 
+        retriever=docsearch.as_retriever(), 
+        # reduce_k_below_max_tokens=True
+    )
+
+
+    # if button:
+    #     prompt = st.text_input("Enter your prompt", value=choice, key="prompt")
+    # else:
+
+
+    prompt = st.text_input("Enter your prompt", value="Imagine you are an analyst looking into Berkshire Hathaway's relationship with GEICO, write a 200 word summary on this aimed at impressing your boss.", key="prompt")
+
+    if prompt:
+        response = qa.run(prompt)
+        st.write(response)
+        # plot_this = st.button("Give me code to plot this")
+
+        # if plot_this:
+        #     followup_llm = OpenAI(temperature=0, model_name="gpt-3.5-turbo")
+        #     plot_code = followup_llm(f"create python code to plot this: {response}")
+        #     st.code(plot_code)
+
+        # put the sources in an expander
+        with st.expander("Sources"):    
+            retriever = docsearch.as_retriever()
+
+            # create a table of the top 3 most similar documents
+            similarity_table = retriever.vectorstore.similarity_search_with_score(prompt, k=3)
+            st.write(similarity_table)
+
+
+    # in an expander, show the documents as a UMAP
+    # with st.expander("Document Embeddings"):
+    #     df = pd.read_parquet("/Users/hamish/Workspace/delphicai/data/chroma-embeddings.parquet")
+    #     umap_mapper = umap.UMAP(n_neighbors=5, min_dist=0.1).fit(df.embedding.tolist())
+    #     p = umap.plot.interactive(umap_mapper, point_size=5)
+        
+    #     # show the plot in streamlit
+    #     st.bokeh_chart(p, use_container_width=True)

data/chroma-collections.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:187e9ab81a8ddc83a4197346733cf576aef5641b17a101e7cada20f766b063b0
+size 557

data/chroma-embeddings.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0b6274fd8c7d2b130d78f70fff66c190823083932ca0c189901ecb04a238e39
+size 5468138

data/index/id_to_uuid_569e1be0-57b0-4ef4-abcc-a40cccd1b383.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad11d5913073c6bd04169cd9b66d8ef210bdefda127b1a285f011244f9955b82
+size 6077

data/index/id_to_uuid_68425e73-3b8d-47fd-9dd0-ceb0ad113836.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d57feb0308972a3ab352191d913af01f63a834a44586d873e64e82ae9c78652a
+size 14031

data/index/index_569e1be0-57b0-4ef4-abcc-a40cccd1b383.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8108b2b89235fcc744b9dada9ca7bd8e4b6268b8fba0c4c6de7c93c6b04779f1
+size 1208276

data/index/index_68425e73-3b8d-47fd-9dd0-ceb0ad113836.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748f63e3c85ff8a065f50516ae0e9e7e0e5681f8b2927773e1234d450c81084f
+size 2756144

data/index/index_metadata_569e1be0-57b0-4ef4-abcc-a40cccd1b383.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ada77f186fa63ea591d1166c8b2d5ba8b9883d0b5cd527e5d5b1f4a3ed64808
+size 73

data/index/index_metadata_68425e73-3b8d-47fd-9dd0-ceb0ad113836.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:571d06c80abef5303ad067a2522a3e22982d39f33b1b03a0edc00e820fcf6170
+size 74

data/index/uuid_to_id_569e1be0-57b0-4ef4-abcc-a40cccd1b383.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc5b7aeb57bbd35985867677f88b1dc7af6e65386786a73d065dc2f117bf9e21
+size 7120

data/index/uuid_to_id_68425e73-3b8d-47fd-9dd0-ceb0ad113836.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec75e2300d30997d654cb028c6f47bccf2e809ddd3dc703386ba9c0bf2142d07
+size 16405