refactor

app.py

@@ -1,27 +1,47 @@
-import streamlit as st
-from utils import get_chroma_client, get_embedding_function
+"""Entry point of streamòit app"""
 
-# streamlit_app.py
-
-import hmac
 import streamlit as st
+import hmac
 import os
-import streamlit.components.v1 as components
-from retrieve_kb import get_current_knowledge_bases, get_knowledge_base_information
-import streamlit as st
 import requests
-import os
 from dotenv import load_dotenv
+import streamlit.components.v1 as components
+
+from utils import get_chroma_client, get_embedding_function
+from retrieve_kb import get_current_knowledge_bases
 
 __import__("pysqlite3")
 import sys
 
+# settings
 sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
-
 st.set_page_config(page_title="Hello", page_icon="👋", layout="wide")
+load_dotenv()
+brian_api_key = os.getenv("BRIAN_API_KEY")
+openai_key = os.getenv("OPENAI_API_KEY")
+
+
+def askbrian_request(prompt, kb, api_key):
+    url = " https://api.brianknows.org/api/v0/agent/knowledge"
+    data = {"prompt": prompt, "kb": kb}
+    headers = {
+        "Content-Type": "application/json",
+        "X-Brian-Api-Key": api_key,  # Include the API key in the headers
+    }
+
+    response = requests.post(url, json=data, headers=headers)
+
+    if response.status_code == 200:
+        return response.json()  # Returns the JSON response if successful
+    else:
+        return (
+            response.status_code,
+            response.text,
+        )  # Returns the status code and error if not successful
 
 
 def show_sidebar():
+    """Shows sidebar with Biran info"""
     # Sidebar
     st.sidebar.header(("About"))
     st.sidebar.markdown(
@@ -73,20 +93,10 @@ if not check_password():
 
 # Main Streamlit app starts here
 client = get_chroma_client()
-default_embedding_function = get_embedding_function()
-
-
-# Function to load a page
-def load_page(page_name):
-    with open(f"pages/{page_name}", "r") as file:
-        exec(file.read(), globals())
-
-
-client = get_chroma_client()
-default_embedding_function = get_embedding_function()
-
+default_embedding_function = get_embedding_function(openai_key=openai_key)
 show_sidebar()
 
+
 col1, col2, col3 = st.columns((1, 4, 1))
 with col2:
     st.image("https://brianknows.org/brian_logo.png", width=300)
@@ -94,39 +104,21 @@ st.write("# Brian Knowledge Base System! 👋")
 
 
 tab1, tab2 = st.tabs(["AskBrian", "BrianApp"])
+
+# Ask Brian Tab
 with tab1:
     st.markdown("## Ask Brian Anything")
     kb_name = "public-knowledge-box"
 
-    load_dotenv()
-    api_key = os.getenv("BRIAN_API_KEY")
-
-    def send_post_request(prompt, kb):
-        url = " https://api.brianknows.org/api/v0/agent/knowledge"
-        data = {"prompt": prompt, "kb": kb}
-        headers = {
-            "Content-Type": "application/json",
-            "X-Brian-Api-Key": api_key,  # Include the API key in the headers
-        }
-
-        response = requests.post(url, json=data, headers=headers)
-
-        if response.status_code == 200:
-            return response.json()  # Returns the JSON response if successful
-        else:
-            return (
-                response.status_code,
-                response.text,
-            )  # Returns the status code and error if not successful
-
     # Example usage:
     kbs = get_current_knowledge_bases(client=client)
     kbs = (kb.name for kb in kbs)
     kb_name = st.selectbox("Select knowledge box", kbs)
     query = st.text_input(label="query")
     if st.button("askbrian"):
-        result = send_post_request(query, kb_name)
+        result = askbrian_request(query, kb_name, brian_api_key)
         st.json(result)
 
+# Brian App embedded Tab
 with tab2:
     components.iframe("https://www.brianknows.org/", height=650, scrolling=True)

generate_kb.py

@@ -13,7 +13,6 @@ from spellchecker import SpellChecker
 
 load_dotenv()
 openai_key = os.getenv("OPENAI_API_KEY")
-openai_key = st.secrets["OPENAI_API_KEY"]
 
 
 def clean_text(text):

pages/create_knowledge_box.py

@@ -1,11 +1,12 @@
 import streamlit as st
-from app import client, default_embedding_function
+from app import client, default_embedding_function, show_sidebar
 import pandas as pd
 from generate_kb import generate_knowledge_box_from_url
 from utils import get_chroma_client
 
 # Title of the app
 st.title("Create a knowledge box from CSV file")
+show_sidebar()
 
 # File uploader widget
 uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"])

pages/delete_knowledge_box⚠️.py

@@ -1,9 +1,10 @@
 import streamlit as st
 from retrieve_kb import get_current_knowledge_bases
-from app import client
+from app import client, show_sidebar
 
 
 st.title("Delete knowledge Base ☠️")
+show_sidebar()
 
 st.title("Get knowledge boxes")
 if st.button("Get current knowledge bases"):

pages/manage_knowledge_box.py

@@ -1,8 +1,7 @@
-from __future__ import unicode_literals
+"""Page to manage kbs"""
+
+from __future__ import unicode_literals  # this should always be the first import
 import streamlit as st
-from retrieve_kb import get_current_knowledge_bases, get_knowledge_base_information
-from generate_kb import add_links_to_knowledge_base
-from app import client, default_embedding_function
 import pandas as pd
 from tempfile import NamedTemporaryFile
 import os
@@ -11,136 +10,17 @@ from openai import OpenAI
 import wave
 from dotenv import load_dotenv
 
+from retrieve_kb import get_current_knowledge_bases, get_knowledge_base_information
+from generate_kb import add_links_to_knowledge_base
+from app import client, default_embedding_function, show_sidebar
+
 
 load_dotenv()
 openai_key = os.getenv("OPENAI_API_KEY")
-
-st.title("Manage collections")
-kbs = get_current_knowledge_bases(client=client)
-kbs = (kb.name for kb in kbs)
-collection_name = st.selectbox("Select knowledge box", kbs)
-info = {}
-collection = None
-
-if "df" not in st.session_state:
-    st.session_state["df"] = pd.DataFrame()
-
-col1, col2 = st.columns(2)
-
-if st.button("Get All"):
-    collection_info, coll, client = get_knowledge_base_information(
-        client=client,
-        embedding_function=default_embedding_function,
-        kb_name=collection_name,
-    )
-    st.session_state["collection"] = coll
-    st.session_state["client"] = client
-    collection = coll
-    # st.write(collection_info)
-    df = pd.DataFrame.from_records(collection_info)
-    df["source"] = df["metadatas"].apply(lambda x: x.get("source", "unkown"))
-    df["title"] = df["metadatas"].apply(lambda x: x.get("title", "unkown"))
-    df = df[["documents", "source", "title", "ids"]]
-    st.session_state["df"] = df
+show_sidebar()
 
 
-if len(st.session_state["df"]) != 0:
-    st.dataframe(st.session_state["df"], width=3_000)
-    unique_df = st.session_state["df"]["source"].unique()
-    st.text(f"unique urls:  {len(unique_df)}")
-    st.dataframe(unique_df)
-
-#############################
-#### REMOVE A SPLIT #########
-#############################
-st.header("Remove a split")
-id = st.text_input("Insert a split id")
-if st.button("Remove Id from collection"):
-    if id in st.session_state["df"]["ids"].values.tolist():
-        res = st.session_state["collection"].delete(ids=[f"{id}"])
-        st.success(f"id {id} deleted")
-    else:
-        st.error(f"id {id} not in kb")
-
-
-#############################
-#### REMOVE URL ############
-#############################
-st.header("Remove url from collection")
-url = st.text_input("remove url")
-if st.button("Remove url from collection"):
-    try:
-        ids = st.session_state["collection"].get(where={"source": url})["ids"]
-        st.session_state["collection"].delete(ids=ids)
-        st.success("deleted")
-    except Exception as e:
-        st.error(str(e))
-
-
-#############################
-########### ADD URL #########
-#############################
-st.header("Add url to existing collection")
-url_text = st.text_input("Insert a url link")
-if st.button("add url to collection"):
-    urls = [url_text]  # put in a list even if only one
-    res = add_links_to_knowledge_base(client=client, kb_name=collection_name, urls=urls)
-    st.write(res)
-
-st.header("Add pdf to existing collection")
-uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
-pdf_optional_link = st.text_input(
-    "Insert a URL link you want to associate with the pdf"
-)
-pdf_title = st.text_input("This title will be displayed as a resource in ask brian")
-if st.button("add pdf"):
-    # Create a temporary file
-    with NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
-        # Write the uploaded PDF to the temporary file
-        tmp_file.write(uploaded_file.getvalue())
-        tmp_path = tmp_file.name
-        print("PATH: ", tmp_path)
-        urls = [tmp_path]
-        res = add_links_to_knowledge_base(
-            client=client,
-            kb_name=collection_name,
-            urls=urls,
-            pdf_optional_link=pdf_optional_link,
-            pdf_title=pdf_title,
-        )
-        st.write(res)
-    # Clean up: delete the temporary file
-    os.remove(tmp_path)
-
-#############################
-########### ADD CSV #########
-#############################
-st.header("Add csv to existing collection")
-uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"])
-df = None
-
-if uploaded_file is not None:
-    try:
-        new_df = pd.read_csv(uploaded_file)
-        st.write("DataFrame:")
-        st.write(new_df)
-    except Exception as e:
-        st.error(str(e))
-    if st.button("add csv urls to collection"):
-        urls = new_df.values.tolist()
-        st.write(urls)
-        res = add_links_to_knowledge_base(
-            client=client, kb_name=collection_name, urls=urls
-        )
-        st.write(res)
-
-
-#############################
-########## YOUTUBE ##########
-#############################
-
-
-def transcribe_audio(audio_path, chunk_length=10000):
+def transcribe_audio(audio_path, openai_key, chunk_length=10000):
     """
     Transcribe audio by breaking it into chunks using wave and numpy.
     :param audio_path: Path to the audio file (e.g., "video.wav").
@@ -148,7 +28,7 @@ def transcribe_audio(audio_path, chunk_length=10000):
     :return: Full transcription of the audio file.
     """
     # Open the wave file
-    client = OpenAI(api_key=open_ai_key)
+    client = OpenAI(api_key=openai_key)
 
     with wave.open(audio_path, "rb") as audio:
         frame_rate = audio.getframerate()
@@ -218,7 +98,7 @@ def download_and_transcribe_youtube(youtube_url):
         video_title = info_dict.get("title", None)
 
         # audio_file = open("video.wav", "rb")
-        text = transcribe_audio("video.wav")
+        text = transcribe_audio(audio_path="video.wav", openai_key=openai_key)
         f_out_path = f"{video_title}.txt"
         with open(f"{video_title}.txt", "w") as f_out:
             f_out.write(text)
@@ -235,19 +115,184 @@ def download_and_transcribe_youtube(youtube_url):
     os.remove("temp_chunk.wav")
 
 
-st.header("Add youtube video to collection")
-st.image(
-    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVsAAACRCAMAAABaFeu5AAAAwFBMVEX////NIB8AAADKAADMGBf//PzNHRzWUVH23d3LCwjjlZUlJSXLDw2np6fi4uLvx8ZycnLPKSfghoY9PT3no6OPj49oaGicnJzZZmXvwsFPT0/66+uvr6/pqanaamr99vbz0tLprq755+fsurrSPDseHh7WVVTu7u5/f3/hjY3fhIPQMzLX19fAwMDbc3K3t7fQ0NB3d3fXW1rUSEguLi5cXFxERETUSUnlnJsUFBSSkpLdeHj119fSNzdeXl41NTXZRimFAAAOiklEQVR4nO2dfXvSPBTGwZRCeTqRjSnIgPIiE3BMQR0K0+//rZ4WaJs7OUlbXtZ2cv/hddmGEH5L05OTk5NC4VWp1HPVdjWoVDaO49RqtVarNdyq76njquH9s/3f7oZbwi3nlt5UKgPvw14lpbR/SkpyCQ4qLrdhv7GqTq6n8+aivBzN6kXTsC0WqThFbNso1uuz0XLc7M6nk8mq0R+6/CsDl/qrwz7YtDqrybpZXj7XbQ6CS8Hl4Mrcqngq7arzKjbcL7C4v4dlzpbl5nqy6rQ2g7SxHCun0Z0FvWkL8WQED8S+Bb7DXe+unLQBHara3P0NRso0NTINt33dWtqYDlB/xuzscvVl2qzeSBtVQrXqLPtgdzKZ2U8bVwL1Frkh68lk5XbayOLKsey0cSWUwVppQ4unfq467U4my8Wou2JpgzpIbJI2uGjlFG0e4LbyitaFm3FzoZ3DsTYQy/Y8eJE3C4GX8ZQ2Pp1q+R0RPLFh2gA1Ghlp4zlKZj1tgGrl+EW2U4Y7bjnf3dYdcUdpI1Spkvdu63bcTdoQFZpYaaM5WvY6bYgKZdgPHlemnc0lNSf/Q4I7KOzXIUog6udGlzihJvy8wVsGjKskv93kP3iG58QfFN6AbuVfe4Ulzgz3mfupxmI6nV7H0zSBdWHOJuEHJ7PTwzWLux9zC+Ru5F/7Bwq8Py/aNj8kJPJ7JBhLjDH/wXPYfKyyrfoLdsue1Gi8/3gUukgNgW0n/gdLSdiW+U+ehe2+V/wCdg9iowX2Zx4Spvy4mV+29nxX92dg91ls9Ae4/fE4dJECX0J+2foD7leA95/Y6J9w++44dFHqAaH8si2y/ZovPvT3Qpu1d0+tzath6+wqx0FB6JnYqwkz4qQavhq2+6YjPsHIQhPtw7HwIrSyqAbGUcbYWtV97fjYY5u/w723R7KLEpgJeWZrd/e1vwN+X/gvvodb34+GF6EF/M4csw2mJw/q5/4RbhEz4tPqGeafOWZrzvx2AUAwYb+pu/Q5hA7GJHPerLEN3IzoMuC/+JNmKD690Lwt2lNvJ0goLDuEe8MEjrAXYFtkvvMAn/yv4fe+hRvSnO3Uagudz2YgmG9XhO0fCX73i7Ct7KvHNxY3qt6poJ9H2rUys1iKXVavF2Hr+PXDxJabIKgHi7NIG/WRM7bBCIb+73BiC5ffnZ3t8BWxDWwcHBQCHy36FyX348nVSYPt01nYBhMzYfr1zb+K/sXzL15Wdevn+WJrXwdfABQ/+Vc/8ld/nB1tYa0zpHLGdh58ARpbvtuAHinOp67uZ74AW9M04my6jLO7lV+T+4/HuPczooOMcN2W7r98/frldD7d8cFsTV5Rl2m2JmPmaPlsM+0sxHBNaaNe9/YUW9rWckFh4MT9s7sG/sWfIoiHb3/9eze3tOl7+/NjoJ9hv397E17+yA81I11v0LHd72reKaSouEyxNdisMfC+oLSZMhU0F/+44bS9Yr12bfXE1O8HPlIU++ju2l/+0hVQK6Ff9w3t2n1HVwAD0CeufPFAtkLIa3AnNOA9Bc5hgi175rbktsv0eMPsFS6DtyfKTm6aXDkAtXXKoGEGrlucr+0lD8hx2PILdNoh9BC2sIe5qmRrsipUUJgTLZEKeeqtFXszICYMPF7bTgheBnDdosM3kBQXkhO2huWILZeNB6NYEQtt1VLAZVwXByfuR5r2XmCa8RKX0xKy7aXD1jbljbii16hoL+WomH1DDHJ4ZnylQMn7GRAUwg0J71VopRdeQraDVNguDYqaEAZsLBVkXbUtqufCXihA9qAmgJM1QeiGTMhWb7KejS25IQw7rllU9VpPG6rdjB9CYHi9Fd5XITOcZUgCYywXbBV77ZoQ4aOPsm8QDQcbBcyC74J/MUSmHGyDD4ZKyJb8+5+drUJ9rn4rapfuUh5y8buB2j2Ov0EhIfROFu8tS8hWHzP+wmy5wd+0dSOCouX43eDEfQSIgWdMWJwkxC9lJmSr3xD5wmy5ddHIbkvtN8REFTAofAbSYWcUSH6+u7vFIFPe75CQrdY1fla2retmtyF0zjBWgqGN5ky6zXkDr/Wlpgs7+G74n/yDBxAUwanxfiUC579cQFlO2A5NZhs2Yzg5CHZeCLufu8wyvNLQL2XTXGD7+41C4YQLMAaP/zu6cFK28h//Rdh29zMrYT9j0BprxV/202aYDPq5NCgIwRXK91ToKYD3XRAKgp6HnLFtKlw7weiPl4OJAroXrkWnjRi4omJLL05y5tYPunQu2IZrkRbQCr4A4iLCr8VapPFMDLjCSNxA4asfX3fhB2GuFkY2JWSrXYp8AbZGl7/uG2FYecjQNFTNIdnimyrQHV2Ac9/AnC4cQfLGFpwG/usJiXNTMOjP0stMChSk2YYPOTjLOIMAmIcQc8aWrsYG65aLvkbTrC44bCS2pGOWc26Bj0HF9pauLgdsDaoaTKhWVbEV16MkthiJuxf37MO4eqX4XDiJyxtbm2Q7JCsR2YrhORJbYVfOTpzrFizg3+F16M9hbNPrYNsiKxHZLqLYCsF1W/GLhTB1UMU7hpOHf4ltM5ItscjIB+L/k2zpSkS2YtyKzBZnWFvxgfgXtkewBX/NVr/4uxe2VRbE6SRmK62GwQrYhe0xbKXlMFgAu7BVppeJwRaD7t4IgfgXtkrFYSs4cTEQ/8JWqThsBScuBuJf2CoVh62QYwXvXdgqFYstWmF478JWqQvbf5Zt2ms6F7Z7ZZ4ttYn+whareR1s04n9SMa2v2iS6oppHcnM2Be2OrZVZtAS45szxlafxf2Me0mSsI2bVpo8FCo9tunEiP4bbNOJbT4XW+oVmB7bdGLyz8XWubA9G1tqO1p6bNPZA3U4WzigWrITqD0q6bFNae/eoWytaqcRqLMU7VvqZM702Ka2n/cwthHxCdT+kwtbrOZAttZZ2B4cs6Q/kyQ1tgfFLMEe/0A6trCYpoq/PZztc47ZCjlL6FPMdGwhekEVIxpGNCRlq03V9QJs6RhRcAwo42+FrADmc4GQji30TxXbsD8nZXtoLqBTsSWrwfhbPm5cF39L7w/UsX2gYamuJ2V7aA4rJVv6cVaxFTaS7avhMn0VwMeMbAUfI0by+9KxhQV21bssXHZXsIVRm2c7OS1bEzMjRLLlcnoVuP0OC/4q508CK6skvIfxL+JLxxZin7kMSbDBJ4yFptmqD5Sg9sofwVbYmR/JlsEmPT+DAo7CYQ5j/3yMncR5j0Ukt9GzxYim8DKkIA4RkGyFMFSerXZRJzFbcVVFwTbc0Yh/i2AGrrC18HQ9cb5OHzuvZQvJQYKe+KCA9Y0ofS9srebZap3jOrZoTDKiFxaUbB0/HZiBr/bg6UdDIbwMDgOx6fRhp1q2mMx1//RjT+SCSomEbm8FtMBW68DV5V5j0Mim7T7I8hq2aj9ve8y8wdIQvILBChMOw96qjinXL74q6BVMLVvhgd5u03tAXFxQ6W+x8L2c7IZnq3WExWc7eGZsIbv41Hv8a02DsScHLwaDv4nJuAutJ4tZZYGdtL2MzHilZSvFld/8FC7wnxB3T3wSywpstY4wLVth8t6mJvPa3BTyJ0KDUHq+e20xX630xJFusAi25A40XrdJCgtsC7rDG7Vs6bxoqIR5P7g5BTnHAs2lbehktmA9W+EQGFl8VtGInEwyW93ZjVq21MKfqGRs+QMjIg+akAYzxaHoEWwV+6l9YcpLugy8EIGtLkmrjq0l2gT7MnHmZYo8P/yic2QyIKndwhTPVwTbgpRBlJeQ2I5MePUIL0Rgq5v06tji6Zq+2rH8CQuaGzTFJusPJM95jCZZMIotJqJA/RJGmSuizJ3an6Bd6dPmFqa8/O16vLga8mj4Hi6AsSnJaiciPxQ95Y1mq06/9ktM4UxsBXzU+Gq0EzM9Wzkis+1arHHYjtkT8d6ZivaqOo/VhsgkSk/LYrBVZbu8kTuPlF7Is37VbHWRNRE5sSWryDLi+RjHhj2TDCa5IWwultlrSCVpVbxcY7ClcwuTJ3BhbX+3HVvNVmfg6nO524JPr+P94rixdoaIokLwsmZUUsbSmmwyuYIuZMJWnfNyL6W2+aM4k4+fW+yNCGALe1p1Rhj6naRAEca7TgaL7V0VW3gzee94k3X5rkt2RfcvMBeJlTom+YYwWYHU1QdedJmCN33lLN1fv9WnHd75U+Lf/mh8/5v7AsxjLtngvBq8JF8vG/kkK+u9+8We8hEEwUqsOeIv7xYMbLZ29h+vjRWpmN1C5U6It1eb2oqSCjMhge4fPnx+//7bh8eIUwjePl5dPcY7alK7nQTiWOQ/gsGMebVRnRfDe3BIF3eOAxUQ45ZdrK/XC6ZMk1/cHUHw1F1Pp+tm3W2D6imTfHBZ0BEnYnjMbItZunlzhAzbtqNPKTG8YrZ2vZ8MtEtd2ibnRaaVNkZS2uXIvOj44fYs0sfl50T0okPqSnJeaWZFxtllQForLB/K6JAQtaEkF4rlTU5FOv94LiSElWVJ+t3oOVAmJw47lY4w/rMgM6tvMk/ayKXsizqKKzsS1/pzJcEVmjXl2lSImRIgNU3yC1e7sJYJNfMK14oVVpKuFvmEmwe03mEhaXM6QGwR/cOyoIbO/Z9JmYwOSsigKs+qZatsyrYy60Yg1JF3d2dWBptneDpGqTNTr/dlSKbNmnGiVDMmZ+qtp2Z4Fc00LFa8VhzomXlV+tPRdvnc5s6LT1veMfb2tlXjqpM2oSPV3gwb6/EsDFGwLNveJebydE6IpsfRWzi3uAAJczTurlf9Wl47LKneoOK0hv3Gqjpddxfj8nI0q2OMB4ff+wPsxCVJ27MK5ZexLEuuxDbrs9FTedGcT6urRn9YcyoDabvDa1ap1Ov12oNKZeM4tVrNZd/vNFz81cnk+vp6ul7Pu82F+4coPy2XI0/L0XL5VB6PF4tmtztfT91Sk8nEhdfp94cttwrH2bgMez0xZftFmdD/D12BwpeL+0AAAAAASUVORK5CYII=",
-    width=200,  # Manually Adjust the width of the image as per requirement
+if "url_list" not in st.session_state:
+    st.session_state["url_list"] = []
+
+
+def list_manager():
+    def add_element():
+        if len(user_input) > 0:
+            st.session_state["url_list"] += [user_input]
+        else:
+            st.warning("Enter text")
+
+    st.text("C'è un bug!!! Cliccare su add due volte!")
+    with st.expander("Add urls"):
+        user_input = st.text_input("Enter a url")
+        add_button = st.button("Add", key="add_button")
+        col1, col2 = st.columns((2))
+        with col1:
+            if add_button:
+                add_element()
+        with col2:
+            if st.button("reset"):
+                st.session_state["url_list"] = []
+        st.write(st.session_state["url_list"])
+
+
+st.title("Manage collections")
+kbs = get_current_knowledge_bases(client=client)
+kbs = (kb.name for kb in kbs)
+collection_name = st.selectbox("Select knowledge box", kbs)
+info = {}
+collection = None
+
+
+if "df" not in st.session_state:
+    st.session_state["df"] = pd.DataFrame()
+
+col1, col2 = st.columns(2)
+
+if st.button("Get All"):
+    collection_info, coll, client = get_knowledge_base_information(
+        client=client,
+        embedding_function=default_embedding_function,
+        kb_name=collection_name,
+    )
+    st.session_state["collection"] = coll
+    st.session_state["client"] = client
+    collection = coll
+
+    df = pd.DataFrame.from_records(collection_info)
+    df["source"] = df["metadatas"].apply(lambda x: x.get("source", "unkown"))
+    df["title"] = df["metadatas"].apply(lambda x: x.get("title", "unkown"))
+    df = df[["documents", "source", "title", "ids"]]
+    st.session_state["df"] = df
+
+if len(st.session_state["df"]) != 0:
+    st.dataframe(st.session_state["df"], width=3_000)
+    unique_df = st.session_state["df"]["source"].unique()
+    st.text(f"unique urls:  {len(unique_df)}")
+    st.dataframe(unique_df)
+else:
+    st.warning(f"{collection_name} KB is empty")
+
+
+tab1, tab2, tab3, tab4, tab5 = st.tabs(
+    ["Remove", "Add URL", "Add CSV", "Add PDF", "Add Youtube"]
 )
 
-video_url = st.text_input("Youtube video url")
-st.text("Aggiungere il video puo impiegare un bel pò. Avvia e vatti a fare una canna")
-if st.button("Add video"):
-    # Create a temporary file
-    # Write the uploaded PDF to the temporary file
-    try:
-        download_and_transcribe_youtube(video_url)
-        st.success("Video Added")
-    except Exception as e:
-        st.error(f"{str(e)}")
+# remove stuff tab
+with tab1:
+    # remove a split
+    st.header("Remove a split")
+    id = st.text_input("Insert a split id")
+    if st.button("Remove Id from collection"):
+        try:
+            if id in st.session_state["df"]["ids"].values.tolist():
+                res = st.session_state["collection"].delete(ids=[f"{id}"])
+                st.success(f"id {id} deleted")
+            else:
+                st.error(f"id {id} not in kb")
+        except Exception as e:
+            st.error(f"{str(e)}")
+
+    # REMOVE URL
+    st.header("Remove url from collection")
+    url = st.text_input("remove url")
+    if st.button("Remove url from collection"):
+        try:
+            ids = st.session_state["collection"].get(where={"source": url})["ids"]
+            st.session_state["collection"].delete(ids=ids)
+            st.success("deleted")
+        except Exception as e:
+            st.error(str(e))
+
+
+# ADD URL
+with tab2:
+    st.header("Add url to existing collection")
+    url_text = st.text_input(
+        "Insert a url link",
+        help="This should be text stored in a webpage like wikipedia. NB notion pages are not supported yet!",
+    )
+    if st.button("add url to collection"):
+        urls = [url_text]  # put in a list even if only one
+        res = add_links_to_knowledge_base(
+            client=client, kb_name=collection_name, urls=urls
+        )
+        st.write(res)
+
+
+# ADD CSV
+with tab3:
+    list_manager()
+
+    # st.header("Add csv to existing collection")
+    # uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"])
+    # df = None
+    # if uploaded_file is not None:
+    #     try:
+    #         new_df = pd.read_csv(uploaded_file)
+    #         st.write("DataFrame:")
+    #         st.write(new_df)
+    #     except Exception as e:
+    #         st.error(str(e))
+    #     if st.button("add csv urls to collection"):
+    #         urls = new_df.values.tolist()
+    #         st.write(urls)
+    if st.button("add csv urls to collection"):
+        res = add_links_to_knowledge_base(
+            client=client, kb_name=collection_name, urls=st.session_state["url_list"]
+        )
+        st.write(res)
+
+
+# Add  PDF
+with tab4:
+    st.header("Add pdf to existing collection")
+    uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
+    pdf_optional_link = st.text_input(
+        "Insert a URL link you want to associate with the pdf"
+    )
+    pdf_title = st.text_input("This title will be displayed as a resource in ask brian")
+    if st.button("add pdf"):
+        # Create a temporary file
+        with NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
+            # Write the uploaded PDF to the temporary file
+            tmp_file.write(uploaded_file.getvalue())
+            tmp_path = tmp_file.name
+            print("PATH: ", tmp_path)
+            urls = [tmp_path]
+            res = add_links_to_knowledge_base(
+                client=client,
+                kb_name=collection_name,
+                urls=urls,
+                pdf_optional_link=pdf_optional_link,
+                pdf_title=pdf_title,
+            )
+            st.write(res)
+        # Clean up: delete the temporary file
+        os.remove(tmp_path)
+
+
+# Add YOUTUBE
+with tab5:
+    st.header("Add youtube video to collection")
+    st.image(
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVsAAACRCAMAAABaFeu5AAAAwFBMVEX////NIB8AAADKAADMGBf//PzNHRzWUVH23d3LCwjjlZUlJSXLDw2np6fi4uLvx8ZycnLPKSfghoY9PT3no6OPj49oaGicnJzZZmXvwsFPT0/66+uvr6/pqanaamr99vbz0tLprq755+fsurrSPDseHh7WVVTu7u5/f3/hjY3fhIPQMzLX19fAwMDbc3K3t7fQ0NB3d3fXW1rUSEguLi5cXFxERETUSUnlnJsUFBSSkpLdeHj119fSNzdeXl41NTXZRimFAAAOiklEQVR4nO2dfXvSPBTGwZRCeTqRjSnIgPIiE3BMQR0K0+//rZ4WaJs7OUlbXtZ2cv/hddmGEH5L05OTk5NC4VWp1HPVdjWoVDaO49RqtVarNdyq76njquH9s/3f7oZbwi3nlt5UKgPvw14lpbR/SkpyCQ4qLrdhv7GqTq6n8+aivBzN6kXTsC0WqThFbNso1uuz0XLc7M6nk8mq0R+6/CsDl/qrwz7YtDqrybpZXj7XbQ6CS8Hl4Mrcqngq7arzKjbcL7C4v4dlzpbl5nqy6rQ2g7SxHCun0Z0FvWkL8WQED8S+Bb7DXe+unLQBHara3P0NRso0NTINt33dWtqYDlB/xuzscvVl2qzeSBtVQrXqLPtgdzKZ2U8bVwL1Frkh68lk5XbayOLKsey0cSWUwVppQ4unfq467U4my8Wou2JpgzpIbJI2uGjlFG0e4LbyitaFm3FzoZ3DsTYQy/Y8eJE3C4GX8ZQ2Pp1q+R0RPLFh2gA1Ghlp4zlKZj1tgGrl+EW2U4Y7bjnf3dYdcUdpI1Spkvdu63bcTdoQFZpYaaM5WvY6bYgKZdgPHlemnc0lNSf/Q4I7KOzXIUog6udGlzihJvy8wVsGjKskv93kP3iG58QfFN6AbuVfe4Ulzgz3mfupxmI6nV7H0zSBdWHOJuEHJ7PTwzWLux9zC+Ru5F/7Bwq8Py/aNj8kJPJ7JBhLjDH/wXPYfKyyrfoLdsue1Gi8/3gUukgNgW0n/gdLSdiW+U+ehe2+V/wCdg9iowX2Zx4Spvy4mV+29nxX92dg91ls9Ae4/fE4dJECX0J+2foD7leA95/Y6J9w++44dFHqAaH8si2y/ZovPvT3Qpu1d0+tzath6+wqx0FB6JnYqwkz4qQavhq2+6YjPsHIQhPtw7HwIrSyqAbGUcbYWtV97fjYY5u/w723R7KLEpgJeWZrd/e1vwN+X/gvvodb34+GF6EF/M4csw2mJw/q5/4RbhEz4tPqGeafOWZrzvx2AUAwYb+pu/Q5hA7GJHPerLEN3IzoMuC/+JNmKD690Lwt2lNvJ0goLDuEe8MEjrAXYFtkvvMAn/yv4fe+hRvSnO3Uagudz2YgmG9XhO0fCX73i7Ct7KvHNxY3qt6poJ9H2rUys1iKXVavF2Hr+PXDxJabIKgHi7NIG/WRM7bBCIb+73BiC5ffnZ3t8BWxDWwcHBQCHy36FyX348nVSYPt01nYBhMzYfr1zb+K/sXzL15Wdevn+WJrXwdfABQ/+Vc/8ld/nB1tYa0zpHLGdh58ARpbvtuAHinOp67uZ74AW9M04my6jLO7lV+T+4/HuPczooOMcN2W7r98/frldD7d8cFsTV5Rl2m2JmPmaPlsM+0sxHBNaaNe9/YUW9rWckFh4MT9s7sG/sWfIoiHb3/9eze3tOl7+/NjoJ9hv397E17+yA81I11v0LHd72reKaSouEyxNdisMfC+oLSZMhU0F/+44bS9Yr12bfXE1O8HPlIU++ju2l/+0hVQK6Ff9w3t2n1HVwAD0CeufPFAtkLIa3AnNOA9Bc5hgi175rbktsv0eMPsFS6DtyfKTm6aXDkAtXXKoGEGrlucr+0lD8hx2PILdNoh9BC2sIe5qmRrsipUUJgTLZEKeeqtFXszICYMPF7bTgheBnDdosM3kBQXkhO2huWILZeNB6NYEQtt1VLAZVwXByfuR5r2XmCa8RKX0xKy7aXD1jbljbii16hoL+WomH1DDHJ4ZnylQMn7GRAUwg0J71VopRdeQraDVNguDYqaEAZsLBVkXbUtqufCXihA9qAmgJM1QeiGTMhWb7KejS25IQw7rllU9VpPG6rdjB9CYHi9Fd5XITOcZUgCYywXbBV77ZoQ4aOPsm8QDQcbBcyC74J/MUSmHGyDD4ZKyJb8+5+drUJ9rn4rapfuUh5y8buB2j2Ov0EhIfROFu8tS8hWHzP+wmy5wd+0dSOCouX43eDEfQSIgWdMWJwkxC9lJmSr3xD5wmy5ddHIbkvtN8REFTAofAbSYWcUSH6+u7vFIFPe75CQrdY1fla2retmtyF0zjBWgqGN5ky6zXkDr/Wlpgs7+G74n/yDBxAUwanxfiUC579cQFlO2A5NZhs2Yzg5CHZeCLufu8wyvNLQL2XTXGD7+41C4YQLMAaP/zu6cFK28h//Rdh29zMrYT9j0BprxV/202aYDPq5NCgIwRXK91ToKYD3XRAKgp6HnLFtKlw7weiPl4OJAroXrkWnjRi4omJLL05y5tYPunQu2IZrkRbQCr4A4iLCr8VapPFMDLjCSNxA4asfX3fhB2GuFkY2JWSrXYp8AbZGl7/uG2FYecjQNFTNIdnimyrQHV2Ac9/AnC4cQfLGFpwG/usJiXNTMOjP0stMChSk2YYPOTjLOIMAmIcQc8aWrsYG65aLvkbTrC44bCS2pGOWc26Bj0HF9pauLgdsDaoaTKhWVbEV16MkthiJuxf37MO4eqX4XDiJyxtbm2Q7JCsR2YrhORJbYVfOTpzrFizg3+F16M9hbNPrYNsiKxHZLqLYCsF1W/GLhTB1UMU7hpOHf4ltM5ItscjIB+L/k2zpSkS2YtyKzBZnWFvxgfgXtkewBX/NVr/4uxe2VRbE6SRmK62GwQrYhe0xbKXlMFgAu7BVppeJwRaD7t4IgfgXtkrFYSs4cTEQ/8JWqThsBScuBuJf2CoVh62QYwXvXdgqFYstWmF478JWqQvbf5Zt2ms6F7Z7ZZ4ttYn+whareR1s04n9SMa2v2iS6oppHcnM2Be2OrZVZtAS45szxlafxf2Me0mSsI2bVpo8FCo9tunEiP4bbNOJbT4XW+oVmB7bdGLyz8XWubA9G1tqO1p6bNPZA3U4WzigWrITqD0q6bFNae/eoWytaqcRqLMU7VvqZM702Ka2n/cwthHxCdT+kwtbrOZAttZZ2B4cs6Q/kyQ1tgfFLMEe/0A6trCYpoq/PZztc47ZCjlL6FPMdGwhekEVIxpGNCRlq03V9QJs6RhRcAwo42+FrADmc4GQji30TxXbsD8nZXtoLqBTsSWrwfhbPm5cF39L7w/UsX2gYamuJ2V7aA4rJVv6cVaxFTaS7avhMn0VwMeMbAUfI0by+9KxhQV21bssXHZXsIVRm2c7OS1bEzMjRLLlcnoVuP0OC/4q508CK6skvIfxL+JLxxZin7kMSbDBJ4yFptmqD5Sg9sofwVbYmR/JlsEmPT+DAo7CYQ5j/3yMncR5j0Ukt9GzxYim8DKkIA4RkGyFMFSerXZRJzFbcVVFwTbc0Yh/i2AGrrC18HQ9cb5OHzuvZQvJQYKe+KCA9Y0ofS9srebZap3jOrZoTDKiFxaUbB0/HZiBr/bg6UdDIbwMDgOx6fRhp1q2mMx1//RjT+SCSomEbm8FtMBW68DV5V5j0Mim7T7I8hq2aj9ve8y8wdIQvILBChMOw96qjinXL74q6BVMLVvhgd5u03tAXFxQ6W+x8L2c7IZnq3WExWc7eGZsIbv41Hv8a02DsScHLwaDv4nJuAutJ4tZZYGdtL2MzHilZSvFld/8FC7wnxB3T3wSywpstY4wLVth8t6mJvPa3BTyJ0KDUHq+e20xX630xJFusAi25A40XrdJCgtsC7rDG7Vs6bxoqIR5P7g5BTnHAs2lbehktmA9W+EQGFl8VtGInEwyW93ZjVq21MKfqGRs+QMjIg+akAYzxaHoEWwV+6l9YcpLugy8EIGtLkmrjq0l2gT7MnHmZYo8P/yic2QyIKndwhTPVwTbgpRBlJeQ2I5MePUIL0Rgq5v06tji6Zq+2rH8CQuaGzTFJusPJM95jCZZMIotJqJA/RJGmSuizJ3an6Bd6dPmFqa8/O16vLga8mj4Hi6AsSnJaiciPxQ95Y1mq06/9ktM4UxsBXzU+Gq0EzM9Wzkis+1arHHYjtkT8d6ZivaqOo/VhsgkSk/LYrBVZbu8kTuPlF7Is37VbHWRNRE5sSWryDLi+RjHhj2TDCa5IWwultlrSCVpVbxcY7ClcwuTJ3BhbX+3HVvNVmfg6nO524JPr+P94rixdoaIokLwsmZUUsbSmmwyuYIuZMJWnfNyL6W2+aM4k4+fW+yNCGALe1p1Rhj6naRAEca7TgaL7V0VW3gzee94k3X5rkt2RfcvMBeJlTom+YYwWYHU1QdedJmCN33lLN1fv9WnHd75U+Lf/mh8/5v7AsxjLtngvBq8JF8vG/kkK+u9+8We8hEEwUqsOeIv7xYMbLZ29h+vjRWpmN1C5U6It1eb2oqSCjMhge4fPnx+//7bh8eIUwjePl5dPcY7alK7nQTiWOQ/gsGMebVRnRfDe3BIF3eOAxUQ45ZdrK/XC6ZMk1/cHUHw1F1Pp+tm3W2D6imTfHBZ0BEnYnjMbItZunlzhAzbtqNPKTG8YrZ2vZ8MtEtd2ibnRaaVNkZS2uXIvOj44fYs0sfl50T0okPqSnJeaWZFxtllQForLB/K6JAQtaEkF4rlTU5FOv94LiSElWVJ+t3oOVAmJw47lY4w/rMgM6tvMk/ayKXsizqKKzsS1/pzJcEVmjXl2lSImRIgNU3yC1e7sJYJNfMK14oVVpKuFvmEmwe03mEhaXM6QGwR/cOyoIbO/Z9JmYwOSsigKs+qZatsyrYy60Yg1JF3d2dWBptneDpGqTNTr/dlSKbNmnGiVDMmZ+qtp2Z4Fc00LFa8VhzomXlV+tPRdvnc5s6LT1veMfb2tlXjqpM2oSPV3gwb6/EsDFGwLNveJebydE6IpsfRWzi3uAAJczTurlf9Wl47LKneoOK0hv3Gqjpddxfj8nI0q2OMB4ff+wPsxCVJ27MK5ZexLEuuxDbrs9FTedGcT6urRn9YcyoDabvDa1ap1Ov12oNKZeM4tVrNZd/vNFz81cnk+vp6ul7Pu82F+4coPy2XI0/L0XL5VB6PF4tmtztfT91Sk8nEhdfp94cttwrH2bgMez0xZftFmdD/D12BwpeL+0AAAAAASUVORK5CYII=",
+        width=200,  # Manually Adjust the width of the image as per requirement
+    )
+
+    video_url = st.text_input("Youtube video url")
+    st.text(
+        "Aggiungere il video puo impiegare un bel pò. Avvia e vatti a fare una canna"
+    )
+    if st.button("Add video"):
+        # Create a temporary file
+        # Write the uploaded PDF to the temporary file
+        try:
+            download_and_transcribe_youtube(video_url)
+            st.success("Video Added")
+        except Exception as e:
+            st.error(f"{str(e)}")

requirements.txt

@@ -14,4 +14,5 @@ librosa
 future
 yt-dlp
 pysqlite3>=0.5.2
-pyspellchecker>=0.8.1
+pyspellchecker>=0.8.1
+beautifulsoup4>=4.12.2

retrieve_kb.py

@@ -1,9 +1,12 @@
 from fastapi import APIRouter
 from utils import get_chroma_client, get_embedding_function
+import os
+from dotenv import load_dotenv
 
-
+load_dotenv()
+openai_key = os.getenv("OPENAI_API_KEY")
 router = APIRouter()
-default_embedding_function = get_embedding_function()
+default_embedding_function = get_embedding_function(openai_key=openai_key)
 
 
 def get_current_knowledge_bases(client):

utils.py

@@ -3,10 +3,10 @@ from chromadb.config import Settings
 import chromadb.utils.embedding_functions as embedding_functions
 from dotenv import load_dotenv
 import streamlit as st
+import os
 
 load_dotenv()
-# openai_key = os.getenv("OPENAI_API_KEY")
-openai_key = st.secrets["OPENAI_API_KEY"]
+openai_key = os.getenv("OPENAI_API_KEY")
 
 
 def get_chroma_client(
@@ -25,7 +25,7 @@ def get_chroma_client(
     return chroma_client
 
 
-def get_embedding_function(model_name="text-embedding-ada-002"):
+def get_embedding_function(openai_key, model_name="text-embedding-ada-002"):
     openai_ef = embedding_functions.OpenAIEmbeddingFunction(
         api_key=openai_key, model_name=model_name
     )