add application file

.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+#venv
+
+*.pkl
+*.csv
+
+.env
+embeddings/
+*bk

.streamlit/config.toml

@@ -0,0 +1,7 @@
+[theme]
+base = "light"
+primaryColor = "#89CFF0"
+backgroundColor = "#E0F7FE"
+secondaryBackgroundColor = "#FFFCE4"
+textColor = "#000000"
+font = "sans serif"

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Current File",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": false
+    }
+  ]
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 yvann-hub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

setup.sh

@@ -0,0 +1,21 @@
+mkdir -p ~/.streamlit/
+
+echo "\
+[general]\n\
+email = \"yvannbarbotts@gmail.com\"\n\
+" > ~/.streamlit/credentials.toml
+
+echo "\
+[server]\n\
+headless = true\n\
+enableCORS=false\n\
+port = $PORT\n\
+\n\
+[theme]\n\
+base = \"light\"\n\
+primaryColor = \"#89CFF0\"\n\
+backgroundColor = \"#E0F7FE\"\n\
+secondaryBackgroundColor = \"#FFFCE4\"\n\
+textColor = \"#000000\"\n\
+font = \"sans serif\"\n\
+" > ~/.streamlit/config.toml

src/Home.py

@@ -0,0 +1,63 @@
+import streamlit as st
+
+
+#Config
+st.set_page_config(layout="wide", page_icon="💬", page_title="Robby | Chat-Bot 🤖")
+
+
+#Contact
+with st.sidebar.expander("📬 Contact"):
+
+    st.write("**GitHub:**",
+"[yvann-hub/Robby-chatbot](https://github.com/yvann-hub/Robby-chatbot)")
+
+    st.write("**Medium:** "
+"[@yvann-hub](https://medium.com/@yvann-hub)")
+
+    st.write("**Twitter:** [@yvann_hub](https://twitter.com/yvann_hub)")
+    st.write("**Mail** : barbot.yvann@gmail.com")
+    st.write("**Created by Yvann**")
+
+
+#Title
+st.markdown(
+    """
+    <h2 style='text-align: center;'>Robby, your data-aware assistant 🤖</h1>
+    """,
+    unsafe_allow_html=True,)
+
+st.markdown("---")
+
+
+#Description
+st.markdown(
+    """ 
+    <h5 style='text-align:center;'>I'm Robby, an intelligent chatbot created by combining 
+    the strengths of Langchain and Streamlit. I use large language models to provide
+    context-sensitive interactions. My goal is to help you better understand your data.
+    I support PDF, TXT, CSV, Youtube transcript 🧠</h5>
+    """,
+    unsafe_allow_html=True)
+st.markdown("---")
+
+
+#Robby's Pages
+st.subheader("🚀 Robby's Pages")
+st.write("""
+- **Robby-Chat**: General Chat on data (PDF, TXT,CSV) with a [vectorstore](https://github.com/facebookresearch/faiss) (index useful parts(max 4) for respond to the user) | works with [ConversationalRetrievalChain](https://python.langchain.com/en/latest/modules/chains/index_examples/chat_vector_db.html)
+- **Robby-Sheet** (beta): Chat on tabular data (CSV) | for precise information | process the whole file | works with [CSV_Agent](https://python.langchain.com/en/latest/modules/agents/toolkits/examples/csv.html) + [PandasAI](https://github.com/gventuri/pandas-ai) for data manipulation and graph creation
+- **Robby-Youtube**: Summarize YouTube videos with [summarize-chain](https://python.langchain.com/en/latest/modules/chains/index_examples/summarize.html)
+""")
+st.markdown("---")
+
+
+#Contributing
+st.markdown("### 🎯 Contributing")
+st.markdown("""
+**Robby is under regular development. Feel free to contribute and help me make it even more data-aware!**
+""", unsafe_allow_html=True)
+
+
+
+
+

src/modules/chatbot.py

@@ -0,0 +1,72 @@
+# fix Error: module 'langchain' has no attribute 'verbose'
+import langchain
+import streamlit as st
+from langchain.callbacks import get_openai_callback
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.prompt import PromptTemplate
+
+langchain.verbose = False
+
+
+class Chatbot:
+    def __init__(self, model_name, temperature, vectors):
+        self.model_name = model_name
+        self.temperature = temperature
+        self.vectors = vectors
+
+    qa_template = """
+        You are a helpful AI assistant named Robby. The user gives you a file its content is represented by the following pieces of context, use them to answer the question at the end.
+        If you don't know the answer, just say you don't know. Do NOT try to make up an answer.
+        If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.
+        Use as much detail as possible when responding.
+
+        context: {context}
+        =========
+        question: {question}
+        ======
+        """
+
+    QA_PROMPT = PromptTemplate(
+        template=qa_template, input_variables=["context", "question"]
+    )
+
+    def conversational_chat(self, query):
+        """
+        Start a conversational chat with a model via Langchain
+        """
+        # llm = ChatOpenAI(model_name=self.model_name, temperature=self.temperature)
+
+        from modules.llm import ChatGLM
+
+        llm = ChatGLM()
+
+        retriever = self.vectors.as_retriever()
+
+        chain = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            retriever=retriever,
+            verbose=True,
+            return_source_documents=True,
+            max_tokens_limit=4097,
+            combine_docs_chain_kwargs={"prompt": self.QA_PROMPT},
+        )
+
+        chain_input = {"question": query, "chat_history": st.session_state["history"]}
+        with get_openai_callback() as cb:
+            result = chain(chain_input)
+
+            st.session_state["history"].append((query, result["answer"]))
+            # count_tokens_chain(chain, chain_input)
+            st.write(
+                f"###### Tokens used in this conversation : {cb.total_tokens} tokens"
+            )
+
+            return result["answer"]
+
+
+def count_tokens_chain(chain, query):
+    with get_openai_callback() as cb:
+        result = chain(query)
+        st.write(f"###### Tokens used in this conversation : {cb.total_tokens} tokens")
+    return result

src/modules/embedder.py

@@ -0,0 +1,87 @@
+import os
+import pickle
+import tempfile
+
+from langchain.document_loaders import PyPDFLoader, TextLoader
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import FAISS
+
+
+class Embedder:
+    def __init__(self):
+        self.PATH = "embeddings"
+        self.createEmbeddingsDir()
+
+    def createEmbeddingsDir(self):
+        """
+        Creates a directory to store the embeddings vectors
+        """
+        if not os.path.exists(self.PATH):
+            os.mkdir(self.PATH)
+
+    def storeDocEmbeds(self, file, original_filename):
+        """
+        Stores document embeddings using Langchain and FAISS
+        """
+        with tempfile.NamedTemporaryFile(mode="wb", delete=False) as tmp_file:
+            tmp_file.write(file)
+            tmp_file_path = tmp_file.name
+
+        def get_file_extension(uploaded_file):
+            file_extension = os.path.splitext(uploaded_file)[1].lower()
+
+            return file_extension
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=2000,
+            chunk_overlap=100,
+            length_function=len,
+        )
+
+        file_extension = get_file_extension(original_filename)
+
+        if file_extension == ".csv":
+            loader = CSVLoader(
+                file_path=tmp_file_path,
+                encoding="utf-8",
+                csv_args={
+                    "delimiter": ",",
+                },
+            )
+            data = loader.load()
+
+        elif file_extension == ".pdf":
+            loader = PyPDFLoader(file_path=tmp_file_path)
+            data = loader.load_and_split(text_splitter)
+
+        elif file_extension == ".txt":
+            loader = TextLoader(file_path=tmp_file_path, encoding="utf-8")
+            data = loader.load_and_split(text_splitter)
+
+        # embeddings = OpenAIEmbeddings()
+        from langchain.embeddings import HuggingFaceEmbeddings
+
+        modelpath = "intfloat/e5-large-v2"
+        embeddings = HuggingFaceEmbeddings(model_name=modelpath)
+
+        vectors = FAISS.from_documents(data, embeddings)
+        os.remove(tmp_file_path)
+
+        # Save the vectors to a pickle file
+        with open(f"{self.PATH}/{original_filename}.pkl", "wb") as f:
+            pickle.dump(vectors, f)
+
+    def getDocEmbeds(self, file, original_filename):
+        """
+        Retrieves document embeddings
+        """
+        if not os.path.isfile(f"{self.PATH}/{original_filename}.pkl"):
+            self.storeDocEmbeds(file, original_filename)
+
+        # Load the vectors from the pickle file
+        with open(f"{self.PATH}/{original_filename}.pkl", "rb") as f:
+            vectors = pickle.load(f)
+
+        return vectors

src/modules/history.py

@@ -0,0 +1,58 @@
+import os
+import streamlit as st
+from streamlit_chat import message
+
+class ChatHistory:
+    
+    def __init__(self):
+        self.history = st.session_state.get("history", [])
+        st.session_state["history"] = self.history
+
+    def default_greeting(self):
+        return "Hey Robby ! 👋"
+
+    def default_prompt(self, topic):
+        return f"Hello ! Ask me anything about {topic} 🤗"
+
+    def initialize_user_history(self):
+        st.session_state["user"] = [self.default_greeting()]
+
+    def initialize_assistant_history(self, uploaded_file):
+        st.session_state["assistant"] = [self.default_prompt(uploaded_file.name)]
+
+    def initialize(self, uploaded_file):
+        if "assistant" not in st.session_state:
+            self.initialize_assistant_history(uploaded_file)
+        if "user" not in st.session_state:
+            self.initialize_user_history()
+
+    def reset(self, uploaded_file):
+        st.session_state["history"] = []
+        
+        self.initialize_user_history()
+        self.initialize_assistant_history(uploaded_file)
+        st.session_state["reset_chat"] = False
+
+    def append(self, mode, message):
+        st.session_state[mode].append(message)
+
+    def generate_messages(self, container):
+        if st.session_state["assistant"]:
+            with container:
+                for i in range(len(st.session_state["assistant"])):
+                    message(
+                        st.session_state["user"][i],
+                        is_user=True,
+                        key=f"history_{i}_user",
+                        avatar_style="big-smile",
+                    )
+                    message(st.session_state["assistant"][i], key=str(i), avatar_style="thumbs")
+
+    def load(self):
+        if os.path.exists(self.history_file):
+            with open(self.history_file, "r") as f:
+                self.history = f.read().splitlines()
+
+    def save(self):
+        with open(self.history_file, "w") as f:
+            f.write("\n".join(self.history))

src/modules/layout.py

@@ -0,0 +1,44 @@
+import streamlit as st
+
+class Layout:
+    
+    def show_header(self, types_files):
+        """
+        Displays the header of the app
+        """
+        st.markdown(
+            f"""
+            <h1 style='text-align: center;'> Ask Robby about your {types_files} files ! 😁</h1>
+            """,
+            unsafe_allow_html=True,
+        )
+
+    def show_api_key_missing(self):
+        """
+        Displays a message if the user has not entered an API key
+        """
+        st.markdown(
+            """
+            <div style='text-align: center;'>
+                <h4>Enter your <a href="https://platform.openai.com/account/api-keys" target="_blank">OpenAI API key</a> to start chatting</h4>
+            </div>
+            """,
+            unsafe_allow_html=True,
+        )
+
+    def prompt_form(self):
+        """
+        Displays the prompt form
+        """
+        with st.form(key="my_form", clear_on_submit=True):
+            user_input = st.text_area(
+                "Query:",
+                placeholder="Ask me anything about the document...",
+                key="input",
+                label_visibility="collapsed",
+            )
+            submit_button = st.form_submit_button(label="Send")
+            
+            is_ready = submit_button and user_input
+        return is_ready, user_input
+    

src/modules/llm.py

@@ -0,0 +1,28 @@
+import json
+from typing import Any, List, Mapping, Optional
+
+import requests
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.base import LLM
+
+url = "https://openai.proxy.onlyyounotothers.top/chat"
+headers = {"Content-Type": "application/json"}
+
+
+class ChatGLM(LLM):
+    @property
+    def _llm_type(self) -> str:
+        return "custom"
+
+    type = "custom"
+
+    # 重写基类方法，根据用户输入的prompt来响应用户，返回字符串
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+    ) -> str:
+        payload = json.dumps({"q": prompt})
+        response = requests.request("POST", url, headers=headers, data=payload)
+        return response.text

src/modules/robby_sheet/table_tool.py

@@ -0,0 +1,73 @@
+import re
+import sys
+from io import BytesIO, StringIO
+
+import matplotlib.pyplot as plt
+import streamlit as st
+from langchain.callbacks import get_openai_callback
+from pandasai import PandasAI
+from pandasai.llm.openai import OpenAI
+from streamlit_chat import message
+
+
+class PandasAgent:
+    @staticmethod
+    def count_tokens_agent(agent, query):
+        """
+        Count the tokens used by the CSV Agent
+        """
+        with get_openai_callback() as cb:
+            result = agent(query)
+            st.write(f"Spent a total of {cb.total_tokens} tokens")
+
+        return result
+
+    def __init__(self):
+        pass
+
+    def get_agent_response(self, uploaded_file_content, query):
+        llm = OpenAI()
+
+        # from modules.llm import ChatGLM
+
+        # llm = ChatGLM()
+        pandas_ai = PandasAI(llm, verbose=True)
+        old_stdout = sys.stdout
+        sys.stdout = captured_output = StringIO()
+
+        response = pandas_ai.run(data_frame=uploaded_file_content, prompt=query)
+        fig = plt.gcf()
+        if fig.get_axes():
+            # Adjust the figure size
+            fig.set_size_inches(12, 6)
+
+            # Adjust the layout tightness
+            plt.tight_layout()
+            buf = BytesIO()
+            fig.savefig(buf, format="png")
+            buf.seek(0)
+            st.image(buf, caption="Generated Plot")
+
+        sys.stdout = old_stdout
+        return response, captured_output
+
+    def process_agent_thoughts(self, captured_output):
+        thoughts = captured_output.getvalue()
+        cleaned_thoughts = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", thoughts)
+        cleaned_thoughts = re.sub(r"\[1m>", "", cleaned_thoughts)
+        return cleaned_thoughts
+
+    def display_agent_thoughts(self, cleaned_thoughts):
+        with st.expander("Display the agent's thoughts"):
+            st.write(cleaned_thoughts)
+
+    def update_chat_history(self, query, result):
+        st.session_state.chat_history.append(("user", query))
+        st.session_state.chat_history.append(("agent", result))
+
+    def display_chat_history(self):
+        for i, (sender, message_text) in enumerate(st.session_state.chat_history):
+            if sender == "user":
+                message(message_text, is_user=True, key=f"{i}_user")
+            else:
+                message(message_text, key=f"{i}")

src/modules/sidebar.py

@@ -0,0 +1,52 @@
+import streamlit as st
+
+class Sidebar:
+
+    MODEL_OPTIONS = ["gpt-3.5-turbo", "gpt-4"]
+    TEMPERATURE_MIN_VALUE = 0.0
+    TEMPERATURE_MAX_VALUE = 1.0
+    TEMPERATURE_DEFAULT_VALUE = 0.0
+    TEMPERATURE_STEP = 0.01
+
+    @staticmethod
+    def about():
+        about = st.sidebar.expander("🧠 About Robby ")
+        sections = [
+            "#### Robby is an AI chatbot with a conversational memory, designed to allow users to discuss their data in a more intuitive way. 📄",
+            "#### It uses large language models to provide users with natural language interactions about user data content. 🌐",
+            "#### Powered by [Langchain](https://github.com/hwchase17/langchain), [OpenAI](https://platform.openai.com/docs/models/gpt-3-5) and [Streamlit](https://github.com/streamlit/streamlit) ⚡",
+            "#### Source code: [yvann-hub/Robby-chatbot](https://github.com/yvann-hub/Robby-chatbot)",
+        ]
+        for section in sections:
+            about.write(section)
+
+    @staticmethod
+    def reset_chat_button():
+        if st.button("Reset chat"):
+            st.session_state["reset_chat"] = True
+        st.session_state.setdefault("reset_chat", False)
+
+    def model_selector(self):
+        model = st.selectbox(label="Model", options=self.MODEL_OPTIONS)
+        st.session_state["model"] = model
+
+    def temperature_slider(self):
+        temperature = st.slider(
+            label="Temperature",
+            min_value=self.TEMPERATURE_MIN_VALUE,
+            max_value=self.TEMPERATURE_MAX_VALUE,
+            value=self.TEMPERATURE_DEFAULT_VALUE,
+            step=self.TEMPERATURE_STEP,
+        )
+        st.session_state["temperature"] = temperature
+        
+    def show_options(self):
+        with st.sidebar.expander("🛠️ Robby's Tools", expanded=False):
+
+            self.reset_chat_button()
+            self.model_selector()
+            self.temperature_slider()
+            st.session_state.setdefault("model", self.MODEL_OPTIONS[0])
+            st.session_state.setdefault("temperature", self.TEMPERATURE_DEFAULT_VALUE)
+
+    

src/modules/utils.py

@@ -0,0 +1,105 @@
+import os
+import pandas as pd
+import streamlit as st
+import pdfplumber
+
+from modules.chatbot import Chatbot
+from modules.embedder import Embedder
+
+class Utilities:
+
+    @staticmethod
+    def load_api_key():
+        """
+        Loads the OpenAI API key from the .env file or 
+        from the user's input and returns it
+        """
+        if not hasattr(st.session_state, "api_key"):
+            st.session_state.api_key = None
+        #you can define your API key in .env directly
+        if os.path.exists(".env") and os.environ.get("OPENAI_API_KEY") is not None:
+            user_api_key = os.environ["OPENAI_API_KEY"]
+            st.sidebar.success("API key loaded from .env", icon="🚀")
+        else:
+            if st.session_state.api_key is not None:
+                user_api_key = st.session_state.api_key
+                st.sidebar.success("API key loaded from previous input", icon="🚀")
+            else:
+                user_api_key = st.sidebar.text_input(
+                    label="#### Your OpenAI API key 👇", placeholder="sk-...", type="password"
+                )
+                if user_api_key:
+                    st.session_state.api_key = user_api_key
+
+        return user_api_key
+
+    
+    @staticmethod
+    def handle_upload(file_types):
+        """
+        Handles and display uploaded_file
+        :param file_types: List of accepted file types, e.g., ["csv", "pdf", "txt"]
+        """
+        uploaded_file = st.sidebar.file_uploader("upload", type=file_types, label_visibility="collapsed")
+        if uploaded_file is not None:
+
+            def show_csv_file(uploaded_file):
+                file_container = st.expander("Your CSV file :")
+                uploaded_file.seek(0)
+                shows = pd.read_csv(uploaded_file)
+                file_container.write(shows)
+
+            def show_pdf_file(uploaded_file):
+                file_container = st.expander("Your PDF file :")
+                with pdfplumber.open(uploaded_file) as pdf:
+                    pdf_text = ""
+                    for page in pdf.pages:
+                        pdf_text += page.extract_text() + "\n\n"
+                file_container.write(pdf_text)
+            
+            def show_txt_file(uploaded_file):
+                file_container = st.expander("Your TXT file:")
+                uploaded_file.seek(0)
+                content = uploaded_file.read().decode("utf-8")
+                file_container.write(content)
+            
+            def get_file_extension(uploaded_file):
+                return os.path.splitext(uploaded_file)[1].lower()
+            
+            file_extension = get_file_extension(uploaded_file.name)
+
+            # Show the contents of the file based on its extension
+            #if file_extension == ".csv" :
+            #    show_csv_file(uploaded_file)
+            if file_extension== ".pdf" : 
+                show_pdf_file(uploaded_file)
+            elif file_extension== ".txt" : 
+                show_txt_file(uploaded_file)
+
+        else:
+            st.session_state["reset_chat"] = True
+
+        #print(uploaded_file)
+        return uploaded_file
+
+    @staticmethod
+    def setup_chatbot(uploaded_file, model, temperature):
+        """
+        Sets up the chatbot with the uploaded file, model, and temperature
+        """
+        embeds = Embedder()
+
+        with st.spinner("Processing..."):
+            uploaded_file.seek(0)
+            file = uploaded_file.read()
+            # Get the document embeddings for the uploaded file
+            vectors = embeds.getDocEmbeds(file, uploaded_file.name)
+
+            # Create a Chatbot instance with the specified model and temperature
+            chatbot = Chatbot(model, temperature,vectors)
+        st.session_state["ready"] = True
+
+        return chatbot
+
+
+    

src/pages/1_📄Robby-Chat.py

@@ -0,0 +1,100 @@
+import os
+import streamlit as st
+from io import StringIO
+import re
+import sys
+from modules.history import ChatHistory
+from modules.layout import Layout
+from modules.utils import Utilities
+from modules.sidebar import Sidebar
+
+#To be able to update the changes made to modules in localhost (press r)
+def reload_module(module_name):
+    import importlib
+    import sys
+    if module_name in sys.modules:
+        importlib.reload(sys.modules[module_name])
+    return sys.modules[module_name]
+
+history_module = reload_module('modules.history')
+layout_module = reload_module('modules.layout')
+utils_module = reload_module('modules.utils')
+sidebar_module = reload_module('modules.sidebar')
+
+ChatHistory = history_module.ChatHistory
+Layout = layout_module.Layout
+Utilities = utils_module.Utilities
+Sidebar = sidebar_module.Sidebar
+
+st.set_page_config(layout="wide", page_icon="💬", page_title="Robby | Chat-Bot 🤖")
+
+# Instantiate the main components
+layout, sidebar, utils = Layout(), Sidebar(), Utilities()
+
+layout.show_header("PDF, TXT, CSV")
+
+user_api_key = utils.load_api_key()
+
+if not user_api_key:
+    layout.show_api_key_missing()
+else:
+    os.environ["OPENAI_API_KEY"] = user_api_key
+
+    uploaded_file = utils.handle_upload(["pdf", "txt", "csv"])
+
+    if uploaded_file:
+
+        # Configure the sidebar
+        sidebar.show_options()
+        sidebar.about()
+
+        # Initialize chat history
+        history = ChatHistory()
+        try:
+            chatbot = utils.setup_chatbot(
+                uploaded_file, st.session_state["model"], st.session_state["temperature"]
+            )
+            st.session_state["chatbot"] = chatbot
+
+            if st.session_state["ready"]:
+                # Create containers for chat responses and user prompts
+                response_container, prompt_container = st.container(), st.container()
+
+                with prompt_container:
+                    # Display the prompt form
+                    is_ready, user_input = layout.prompt_form()
+
+                    # Initialize the chat history
+                    history.initialize(uploaded_file)
+
+                    # Reset the chat history if button clicked
+                    if st.session_state["reset_chat"]:
+                        history.reset(uploaded_file)
+
+                    if is_ready:
+                        # Update the chat history and display the chat messages
+                        history.append("user", user_input)
+
+                        old_stdout = sys.stdout
+                        sys.stdout = captured_output = StringIO()
+
+                        output = st.session_state["chatbot"].conversational_chat(user_input)
+
+                        sys.stdout = old_stdout
+
+                        history.append("assistant", output)
+
+                        # Clean up the agent's thoughts to remove unwanted characters
+                        thoughts = captured_output.getvalue()
+                        cleaned_thoughts = re.sub(r'\x1b\[[0-9;]*[a-zA-Z]', '', thoughts)
+                        cleaned_thoughts = re.sub(r'\[1m>', '', cleaned_thoughts)
+
+                        # Display the agent's thoughts
+                        with st.expander("Display the agent's thoughts"):
+                            st.write(cleaned_thoughts)
+
+                history.generate_messages(response_container)
+        except Exception as e:
+            st.error(f"Error: {str(e)}")
+
+

src/pages/2_📊 Robby-Sheet (beta).py

@@ -0,0 +1,77 @@
+import os
+import importlib
+import sys
+import pandas as pd
+import streamlit as st
+from io import BytesIO
+from modules.robby_sheet.table_tool import PandasAgent
+from modules.layout import Layout
+from modules.utils import Utilities
+from modules.sidebar import Sidebar
+
+def reload_module(module_name):
+    """For update changes
+    made to modules in localhost (press r)"""
+
+    if module_name in sys.modules:
+        importlib.reload(sys.modules[module_name])
+    return sys.modules[module_name]
+
+table_tool_module = reload_module('modules.robby_sheet.table_tool')
+layout_module = reload_module('modules.layout')
+utils_module = reload_module('modules.utils')
+sidebar_module = reload_module('modules.sidebar')
+
+
+st.set_page_config(layout="wide", page_icon="💬", page_title="Robby | Chat-Bot 🤖")
+
+layout, sidebar, utils = Layout(), Sidebar(), Utilities()
+
+layout.show_header("CSV, Excel")
+
+user_api_key = utils.load_api_key()
+os.environ["OPENAI_API_KEY"] = user_api_key
+
+
+if not user_api_key:
+    layout.show_api_key_missing()
+
+else:
+    st.session_state.setdefault("reset_chat", False)
+
+    uploaded_file = utils.handle_upload(["csv", "xlsx"])
+
+    if uploaded_file:
+        sidebar.about()
+        
+        uploaded_file_content = BytesIO(uploaded_file.getvalue())
+        if uploaded_file.type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" or uploaded_file.type == "application/vnd.ms-excel":
+            df = pd.read_excel(uploaded_file_content)
+        else:
+            df = pd.read_csv(uploaded_file_content)
+
+        st.session_state.df = df
+
+        if "chat_history" not in st.session_state:
+            st.session_state["chat_history"] = []
+        csv_agent = PandasAgent()
+
+        with st.form(key="query"):
+
+            query = st.text_input("Ask [PandasAI](https://github.com/gventuri/pandas-ai) (look the pandas-AI read-me for how use it)", value="", type="default", 
+                placeholder="e-g : How many rows ? "
+                )
+            submitted_query = st.form_submit_button("Submit")
+            reset_chat_button = st.form_submit_button("Reset Chat")
+            if reset_chat_button:
+                st.session_state["chat_history"] = []
+        if submitted_query:
+            result, captured_output = csv_agent.get_agent_response(df, query)
+            cleaned_thoughts = csv_agent.process_agent_thoughts(captured_output)
+            csv_agent.display_agent_thoughts(cleaned_thoughts)
+            csv_agent.update_chat_history(query, result)
+            csv_agent.display_chat_history()
+        if st.session_state.df is not None:
+            st.subheader("Current dataframe:")
+            st.write(st.session_state.df)
+

src/pages/3_🎬 Robby-Youtube.py

@@ -0,0 +1,71 @@
+import os
+import streamlit as st
+import re
+from modules.layout import Layout
+from modules.utils import Utilities
+from modules.sidebar import Sidebar
+from youtube_transcript_api import YouTubeTranscriptApi
+from langchain.chains.summarize import load_summarize_chain
+from langchain.chains import AnalyzeDocumentChain
+from youtube_transcript_api import YouTubeTranscriptApi
+from langchain.llms import OpenAI
+import os
+from langchain.text_splitter import CharacterTextSplitter
+
+st.set_page_config(layout="wide", page_icon="💬", page_title="Robby | Chat-Bot 🤖")
+
+# Instantiate the main components
+layout, sidebar, utils = Layout(), Sidebar(), Utilities()
+
+st.markdown(
+    f"""
+    <h1 style='text-align: center;'> Ask Robby to summarize youtube video ! 😁</h1>
+    """,
+    unsafe_allow_html=True,
+)
+
+user_api_key = utils.load_api_key()
+
+sidebar.about()
+
+if not user_api_key:
+    layout.show_api_key_missing()
+
+else:
+    os.environ["OPENAI_API_KEY"] = user_api_key
+
+    script_docs = []
+
+    def get_youtube_id(url):
+        video_id = None
+        match = re.search(r"(?<=v=)[^&#]+", url)
+        if match :
+            video_id = match.group()
+        else : 
+            match = re.search(r"(?<=youtu.be/)[^&#]+", url)
+            if match :
+                video_id = match.group()
+        return video_id
+
+    video_url = st.text_input(placeholder="Enter Youtube Video URL", label_visibility="hidden", label =" ")
+    if video_url :
+        video_id = get_youtube_id(video_url)
+
+        if video_id != "":
+            t = YouTubeTranscriptApi.get_transcript(video_id, languages=('en','fr','es', 'zh-cn', 'hi', 'ar', 'bn', 'ru', 'pt', 'sw' ))
+            finalString = ""
+            for item in t:
+                text = item['text']
+                finalString += text + " "
+
+            text_splitter = CharacterTextSplitter()
+            chunks = text_splitter.split_text(finalString)
+
+            summary_chain = load_summarize_chain(OpenAI(temperature=0),
+                                            chain_type="map_reduce",verbose=True)
+            
+            summarize_document_chain = AnalyzeDocumentChain(combine_docs_chain=summary_chain)
+
+            answer = summarize_document_chain.run(chunks)
+
+            st.subheader(answer)

tuto_chatbot_csv.py

@@ -0,0 +1,73 @@
+#pip install streamlit langchain openai faiss-cpu tiktoken
+
+import streamlit as st
+from streamlit_chat import message
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import ConversationalRetrievalChain
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.vectorstores import FAISS
+import tempfile
+
+
+user_api_key = st.sidebar.text_input(
+    label="#### Your OpenAI API key 👇",
+    placeholder="Paste your openAI API key, sk-",
+    type="password")
+
+uploaded_file = st.sidebar.file_uploader("upload", type="csv")
+
+if uploaded_file :
+    with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+        tmp_file.write(uploaded_file.getvalue())
+        tmp_file_path = tmp_file.name
+
+    loader = CSVLoader(file_path=tmp_file_path, encoding="utf-8")
+    data = loader.load()
+
+    embeddings = OpenAIEmbeddings()
+    vectors = FAISS.from_documents(data, embeddings)
+
+    chain = ConversationalRetrievalChain.from_llm(llm = ChatOpenAI(temperature=0.0,model_name='gpt-3.5-turbo', openai_api_key=user_api_key),
+                                                                      retriever=vectors.as_retriever())
+
+    def conversational_chat(query):
+        
+        result = chain({"question": query, "chat_history": st.session_state['history']})
+        st.session_state['history'].append((query, result["answer"]))
+        
+        return result["answer"]
+    
+    if 'history' not in st.session_state:
+        st.session_state['history'] = []
+
+    if 'generated' not in st.session_state:
+        st.session_state['generated'] = ["Hello ! Ask me anything about " + uploaded_file.name + " 🤗"]
+
+    if 'past' not in st.session_state:
+        st.session_state['past'] = ["Hey ! 👋"]
+        
+    #container for the chat history
+    response_container = st.container()
+    #container for the user's text input
+    container = st.container()
+
+    with container:
+        with st.form(key='my_form', clear_on_submit=True):
+            
+            user_input = st.text_input("Query:", placeholder="Talk about your csv data here (:", key='input')
+            submit_button = st.form_submit_button(label='Send')
+            
+        if submit_button and user_input:
+            output = conversational_chat(user_input)
+            
+            st.session_state['past'].append(user_input)
+            st.session_state['generated'].append(output)
+
+    if st.session_state['generated']:
+        with response_container:
+            for i in range(len(st.session_state['generated'])):
+                message(st.session_state["past"][i], is_user=True, key=str(i) + '_user', avatar_style="big-smile")
+                message(st.session_state["generated"][i], key=str(i), avatar_style="thumbs")
+                
+#streamlit run tuto_chatbot_csv.py