move elements

.dockerignore

@@ -0,0 +1,12 @@
+./docker/zeppelin/logs/*
+*.openaiapikey*
+*.log
+*.log.*
+*__pycache__*
+root
+*.ipynb_checkpoints*
+.vscode
+/app/mymodels
+/app/.cache
+/app/VectorStore
+*chroma-embeddings.parquet*

Dockerfile

@@ -30,8 +30,9 @@ COPY ./requirements.txt requirements.txt
 RUN pip install -r requirements.txt
 COPY ./app .
 #RUN python load_docs.py
-RUN --mount=type=secret,id=OPENAI_API_KEY \
-  cat /run/secrets/OPENAI_API_KEY > .openaiapikey
+#RUN --mount=type=secret,id=OPENAI_API_KEY \
+#  cat /run/secrets/OPENAI_API_KEY > .openaiapikey
+RUN echo "" > .openaiapikey
 RUN mkdir /.cache
 RUN mkdir /nltk_data
 RUN mkdir /VectorStore
@@ -39,6 +40,7 @@ RUN mkdir /app/.cache
 RUN ls -la
 RUN python run.py
 RUN chmod 777 /VectorStore
+RUN chmod 777 /mymodels
 RUN chmod 777 /nltk_data
 RUN chmod 777 /.cache
 RUN chmod 777 /app/.cache

app/app.py

@@ -1,7 +1,9 @@
 import streamlit as st
 import load_model
 import utils as ut
+import elements as el
 import os
+import torch
 
 persist_directory = load_model.persist_directory
 st.title('myRetrievalGPT')
@@ -11,7 +13,7 @@ st.markdown('*Let $\phi$ be a word embedding mapping $W$ → $\mathbb{R}^n$ w
 
 agree = st.checkbox('Load new Documents')
 if agree:
-    ut.load_files()
+    el.load_files()
 else:
 
     import torch

app/elements.py

@@ -0,0 +1,67 @@
+
+import streamlit as st
+from langchain.docstore.document import Document
+from chromadb.config import Settings
+from load_model import load_embedding
+from load_vectors import load_from_file, load_and_split, create_and_add, load_from_web
+from utils import retrieve_collections, get_chroma_client
+
+def llm_module():
+    pass
+
+def load_files():
+    
+    client = get_chroma_client()
+
+    option = st.radio(
+        "",
+        options=["Add Documents", "Start new collection"],
+    )
+
+    if option == "Add Documents":
+        collections = retrieve_collections()
+        selected_collection = st.selectbox(
+            'Add to exsisting collection or create a new one',
+            collections )
+        if st.button('Delete Collection (⚠️ This is destructive and not reversible)'):
+            client.delete_collection(name=selected_collection["name"])
+            #retrieve_collections.clear()
+            collections = retrieve_collections()
+
+        if selected_collection:
+            st.write("Selected Vectorstore:", selected_collection)
+        option = st.radio(
+            "",
+            options=["Upload Files from Local", "Upload Files from Web"],
+        )
+        if option == "Upload Files from Local":
+            st.write('Source Documents:')
+            uploaded_files = st.file_uploader("Choose a PDF file", accept_multiple_files=True)
+            chunk_size = st.text_area('chunk Size:', 1000)
+
+            if st.button('Upload'):
+                docs = load_from_file(uploaded_files)
+                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
+                vec1 = create_and_add(selected_collection["name"], sub_docs, selected_collection['model_name'], selected_collection['metadata'])
+                st.write("Upload succesful")
+        else:
+            st.write('Urls of Source Documents (Comma separated):')
+            urls = chunk_size = st.text_area('Urls:', '')
+            chunk_size = st.text_area('chunk Size:', 1000)
+            urls = urls.replace(",", "" ).replace('"', "" ).split(',')
+
+            if st.button('Upload'):
+                docs = load_from_web(urls)     
+                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
+                vec2 = create_and_add(selected_collection["name"], sub_docs, selected_collection['model_name'], selected_collection['metadata'])
+                st.write("Upload succesful")
+    else:
+        collection = st.text_area('Name of your new collection:', '')
+        model_name = st.text_area('Choose the embedding function:', "hkunlp/instructor-large")
+        if st.button('Create'):
+            if len(collection)>3:
+                ef = load_embedding(model_name)
+                metadata= {"loaded_docs":[], "Subject":"Terms Example", "model_name": ef.model_name}
+                client.create_collection(collection, embedding_function=ef, metadata=metadata) 
+                # retrieve_collections.clear()
+                st.write("Collection " +collection+" succesfully created.")

app/load_model.py

@@ -82,8 +82,8 @@ def load_gpu_model(used_model):
     return llm
 
 #@st.cache_resource
-def load_openai_model():
-    return OpenAI(temperature=0.9)
+def load_openai_model(temperature=0.9):
+    return OpenAI(temperature=temperature)
 
 @st.cache_resource
 def load_openai_embedding():

app/load_vectors.py

@@ -101,4 +101,21 @@ def load_from_web(urls, cache=True):
 def load_and_split(docs, chunk_size=700):
     text_splitter = NLTKTextSplitter(chunk_size=chunk_size, chunk_overlap=0)
     sub_docs = text_splitter.split_documents(docs)
-    return sub_docs
+    return sub_docs
+
+def metadata_generator(doc, llm,max_token=4000):
+    #query = f"Document = {doc.page_content[1:max_token]} -> Respond a python code using a dict filling xxxx like {{'document_type': xxxx, 'summary (max. 30 letters)':'xxxx'}} resond at leat 10 letter"
+    query = f"""
+    Cluster the following Input document into topic categories based on patterns seen within the text. Also mention reasoning behind how these categories were defined.
+Output format:
+{{
+"DOCUMENT TYPE": "",
+"SUMMARY": [],
+"REASONING": ""
+}}
+
+Input document:
+{doc.page_content[1:max_token]}
+Output:
+    """
+    return llm(query)

app/run.py

@@ -5,7 +5,7 @@ import pathlib
 
 from load_model import load_embedding
 from utils import get_chroma_client
-from load_vectors import load_from_web, create_and_add, load_and_split
+from load_vectors import load_from_web, create_and_add, load_and_split, metadata_generator
 
 current_path = str( pathlib.Path(__file__).parent.resolve() )
 with open(current_path+'/.openaiapikey', 'r') as reader:
@@ -21,12 +21,12 @@ llm= load_model.load_openai_model()
 client = get_chroma_client()
 client.reset()
 ef = load_embedding("hkunlp/instructor-large")
-collection_name="axaterms"
-metadata= {"loaded_docs":[], "Subject":"AXA Terms", "model_name": ef.model_name}
+collection_name="papers"
+metadata= {"loaded_docs":[], "Subject":"Heikos Papers", "model_name": ef.model_name}
 selected_collection = client.create_collection(collection_name, embedding_function=ef, metadata=metadata) 
 
 docs_tarifs= [
-    "https://www.axa.de/site/axa-de/get/documents_E1883536226/axade/medien/privatkunden/haus-und-wohnen/hausratversicherung/hausrat-versicherungsbedingungen-L.pdf",
+    "https://edoc.hu-berlin.de/bitstream/handle/18452/5294/33.pdf",    
 ]
 
 # %%
@@ -40,7 +40,18 @@ docs = load_from_web(docs_tarifs)
 sub_docs = load_and_split(docs, chunk_size=1000)
 create_and_add(collection_name, sub_docs, model_name, metadata)
 
+
+
 # %%
-chain = load_model.create_chain(llm, collection=collection_name, model_name=model_name, metadata=metadata)
+#chain = load_model.create_chain(llm, collection=collection_name, model_name=model_name, metadata=metadata)
 #result = chain({"query": "Ist mein Kinderwagen bei einem Leitungswasserschaden mitversichert?"})
 #print(result)
+#llm= load_model.load_openai_model(temperature=0.1)
+
+#llm= load_model.load_cpu_model()
+
+#meta= metadata_generator(docs[0], llm)
+# %%
+#print(meta)
+
+# %%

app/utils.py

@@ -27,66 +27,9 @@ def get_chroma_client():
     return chromadb.Client(Settings(chroma_db_impl="duckdb+parquet",
                                     persist_directory=persist_directory
                                 ))
-#@st.cache_data
+@st.cache_data
 def retrieve_collections():
     client = get_chroma_client()
     all_collections = client.list_collections()
     collections = tuple( [{'name': collection.name, 'model_name': collection.metadata['model_name'], "metadata": collection.metadata} for collection in all_collections] )
-    return collections
-
-def load_files():
-    
-    client = get_chroma_client()
-
-    option = st.radio(
-        "",
-        options=["Add Documents", "Start new collection"],
-    )
-
-    if option == "Add Documents":
-        collections = retrieve_collections()
-        selected_collection = st.selectbox(
-            'Add to exsisting collection or create a new one',
-            collections )
-        if st.button('Delete Collection (⚠️ This is destructive and not reversible)'):
-            client.delete_collection(name=selected_collection["name"])
-            #retrieve_collections.clear()
-            collections = retrieve_collections()
-
-        if selected_collection:
-            st.write("Selected Vectorstore:", selected_collection)
-        option = st.radio(
-            "",
-            options=["Upload Files from Local", "Upload Files from Web"],
-        )
-        if option == "Upload Files from Local":
-            st.write('Source Documents:')
-            uploaded_files = st.file_uploader("Choose a PDF file", accept_multiple_files=True)
-            chunk_size = st.text_area('chunk Size:', 1000)
-
-            if st.button('Upload'):
-                docs = load_from_file(uploaded_files)
-                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
-                vec1 = create_and_add(selected_collection["name"], sub_docs, selected_collection['model_name'], selected_collection['metadata'])
-                st.write("Upload succesful")
-        else:
-            st.write('Urls of Source Documents (Comma separated):')
-            urls = chunk_size = st.text_area('Urls:', '')
-            chunk_size = st.text_area('chunk Size:', 1000)
-            urls = urls.replace(",", "" ).replace('"', "" ).split(',')
-
-            if st.button('Upload'):
-                docs = load_from_web(urls)     
-                sub_docs = load_and_split(docs, chunk_size=int(chunk_size))
-                vec2 = create_and_add(selected_collection["name"], sub_docs, selected_collection['model_name'], selected_collection['metadata'])
-                st.write("Upload succesful")
-    else:
-        collection = st.text_area('Name of your new collection:', '')
-        model_name = st.text_area('Choose the embedding function:', "hkunlp/instructor-large")
-        if st.button('Create'):
-            if len(collection)>3:
-                ef = load_embedding(model_name)
-                metadata= {"loaded_docs":[], "Subject":"Terms Example", "model_name": ef.model_name}
-                client.create_collection(collection, embedding_function=ef, metadata=metadata) 
-                # retrieve_collections.clear()
-                st.write("Collection " +collection+" succesfully created.")
+    return collections

docker-compose.yaml

@@ -1,6 +1,7 @@
 version: "3.9"
 services:
   streamlit_app:
+    image: myretrievalgpt
     build: .
     tty: true
     ports:
@@ -11,11 +12,13 @@ services:
           devices:
             - capabilities: [gpu]
   dev_app:
-    image: tensorflow/tensorflow:latest-gpu
+    image: myretrievalgpt
     tty: true
     volumes:
       - ./app:/app
       - ./root:/root
+    depends_on:
+      - streamlit_app 
     deploy:
       resources:
         reservations: