Upload 22 files

DV-AGENT/agent_chain.py

@@ -0,0 +1,74 @@
+from langchain.agents import AgentExecutor, create_react_agent
+from langchain.prompts import PromptTemplate
+from memory import memory
+from tools import zeroshot_tools
+import pandas as pd
+import os
+import streamlit as st
+#from langchain_community.llms import HuggingFaceHub
+from typing import List
+from langchain_groq import ChatGroq
+from dotenv import load_dotenv
+load_dotenv()
+
+groq_api_key = os.getenv("GROQ_API_KEY")
+
+llm1 = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
+
+def read_first_3_rows():
+    dataset_path = "dataset.csv"
+    try:
+        df = pd.read_csv(dataset_path)
+        first_3_rows = df.head(3).to_string(index=False)
+    except FileNotFoundError:
+        first_3_rows = "Error: Dataset file not found."
+
+    return first_3_rows
+
+
+def get_agent_chain():
+
+
+    dataset_first_3_rows = read_first_3_rows()
+
+    prompt = PromptTemplate(
+
+    input_variables = ['agent_scratchpad', 'chat_history', 'input', 'tool_names', 'tools'],
+    template = ( f"""
+You are a helpful assistant that can help users explore a dataset.
+First 3 rows of the dataset:
+{dataset_first_3_rows}
+===="""
+"""
+TOOLS:
+------
+You has access to the following tools:
+
+{tools}
+
+To use a tool, please use the following format:
+
+Thought: Do I need to use a tool? Yes
+Action: the action to take, should be one of [{tool_names}]
+Action Input: the input to the action
+Observation: the result of the action
+
+When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:
+
+Thought: Do I need to use a tool? No
+Final Answer: [your response here]
+
+Begin!
+
+New input: {input}
+{agent_scratchpad}"""
+    )
+
+    )
+
+
+    conversational_agent_llm = llm1
+    #conversational_agent_llm = ChatOpenAI(model="gpt-3.5-turbo-16k", temperature=temperature, streaming=True)
+    conversational_agent = create_react_agent(conversational_agent_llm, zeroshot_tools, prompt)
+    room_selection_chain = AgentExecutor(agent=conversational_agent, tools=zeroshot_tools, verbose=True, memory=memory, handle_parsing_errors=True, max_iterations=4)
+    return room_selection_chain

DV-AGENT/app.py

@@ -0,0 +1,105 @@
+import streamlit as st
+from streamlit_option_menu import option_menu
+from memory import memory_storage
+from agent_chain import get_agent_chain
+from default_text import default_text4
+from generate_plot import generate_plot, retry_generate_plot
+from markup import app_intro, how_use_intro
+from modules import replace_default_dataset, save_uploaded_dataset
+import pandas as pd
+import os
+
+if 'error' not in st.session_state:
+    st.session_state['error'] = []
+
+def tab1():
+
+    col1, col2 = st.columns([1, 2])
+    with col1:
+        st.image("image.jpg", use_column_width=True)
+    with col2:
+        st.markdown(app_intro(), unsafe_allow_html=True)
+    st.markdown(how_use_intro(),unsafe_allow_html=True) 
+
+    
+    #st.markdown("<p style='font-size: 14px; color: #777;'>Disclaimer: This app is a proof-of-concept and may not be suitable for real-world decisions. During the Hackthon period usage information are being recorded using Langsmith</p>", unsafe_allow_html=True)
+
+
+
+def tab2():
+
+    dataset_option = st.radio("Select Dataset Option", ("Default", "Upload"))
+    
+    if dataset_option == "Default":
+        if st.button("Use Default Dataset"):
+            replace_default_dataset()
+    else:
+        uploaded_file = st.file_uploader("Upload a CSV file", type=["csv"])
+        if uploaded_file:
+            save_uploaded_dataset(uploaded_file)
+
+    st.header("🗣️ Chat")
+
+    for i, msg in enumerate(memory_storage.messages):
+        name = "user" if i % 2 == 0 else "assistant"
+        st.chat_message(name).markdown(msg.content)
+
+    if user_input := st.chat_input("User Input"):
+
+        with st.chat_message("user"):
+            st.markdown(user_input)
+
+        with st.spinner("Generating Response..."):
+
+            with st.chat_message("assistant"):
+                zeroshot_agent_chain = get_agent_chain()
+                response = zeroshot_agent_chain({"input": user_input})
+
+                answer = response['output']
+                st.markdown(answer)
+
+    
+    if st.sidebar.button("Clear Chat History"):
+        memory_storage.clear()
+
+
+def tab4():
+
+    try:
+        df = pd.read_csv("dataset.csv")
+
+        st.header("Dataset Content")
+        st.dataframe(df)
+
+    except FileNotFoundError:
+        st.error("File 'dataset.csv' not found in the current directory.")
+
+    except pd.errors.EmptyDataError:
+        st.error("File 'dataset.csv' is empty.")
+
+    except pd.errors.ParserError:
+        st.error("File 'dataset.csv' could not be parsed as a CSV file.")
+
+def main():
+    st.set_page_config(page_title="NaturalViz", page_icon=":memo:", layout="wide")
+
+    #os.environ['LANGCHAIN_TRACING_V2'] = "true"
+    #os.environ['LANGCHAIN_API_KEY'] == st.secrets['LANGCHAIN_API_KEY']
+
+    tabs = ["Intro","Chat", "View Dataset"]
+
+    with st.sidebar:
+
+        current_tab = option_menu("Select a Tab", tabs, menu_icon="cast")
+
+    tab_functions = {
+    "Intro": tab1,
+    "Chat": tab2,
+    "View Dataset": tab4,
+    }
+
+    if current_tab in tab_functions:
+        tab_functions[current_tab]()
+
+if __name__ == "__main__":
+    main()

DV-AGENT/app2.py

@@ -0,0 +1,21 @@
+import altair as alt
+import pandas as pd
+import streamlit as st
+
+# Read the dataset
+df = pd.read_csv('dataset.csv')
+
+# Calculate the sum of calories
+calories_sum = df['calories'].sum()
+
+# Create a dataframe with the sum of calories
+calories_df = pd.DataFrame({'calories': [calories_sum]}, index=['Total'])
+
+# Create the chart
+chart = alt.Chart(calories_df).mark_bar().encode(
+    x='calories:Q',
+    y='sum(calories):Q'
+)
+
+# Display the chart
+st.altair_chart(chart, use_container_width=True)

DV-AGENT/default_text.py

@@ -0,0 +1 @@
+default_text4 = """Total number of rows"""

DV-AGENT/generate_plot.py

@@ -0,0 +1,121 @@
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+import pandas as pd
+import os
+import streamlit as st
+#from langchain_community.llms import HuggingFaceHub
+from typing import List
+from langchain_groq import ChatGroq
+from dotenv import load_dotenv
+load_dotenv()
+
+groq_api_key = os.getenv("GROQ_API_KEY")
+
+llm1 = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
+
+
+def read_first_3_rows():
+    dataset_path = "dataset.csv"
+    try:
+        df = pd.read_csv(dataset_path)
+        first_3_rows = df.head(3).to_string(index=False)
+    except FileNotFoundError:
+        first_3_rows = "Error: Dataset file not found."
+
+    return first_3_rows
+
+def generate_plot(question):
+    dataset_first_3_rows = read_first_3_rows()
+
+    GENERATE_PLOT_TEMPLATE_PREFIX = """You are a high skilled visualization assistant that can modify a provided visualization code based on a set of instructions. You MUST return a full program. DO NOT include any preamble text. Do not include explanations or prose.
+    First 3 rows of the dataset:"""
+
+    DATASET = f"{dataset_first_3_rows}"
+
+    GENERATE_PLOT_TEMPLATE_SUFIX = """
+Question:
+{question}
+
+# comment Example for protein count of different products:
+
+import altair as alt
+import pandas as pd
+import streamlit as st
+
+#  comment Read the dataset
+df = pd.read_csv('dataset.csv')
+
+#  comment Calculate the protein count of different products
+product_protein = df.groupby('name')['protein'].sum().reset_index()
+
+# comment Create the chart
+chart = alt.Chart(product_protein).mark_bar().encode(
+    x=alt.X('name:N', title='Product Name'),
+    y=alt.Y('protein:Q', title='Protein Count')
+)
+
+# comment Display the chart
+st.altair_chart(chart, use_container_width=True)
+
+"""
+
+    template = GENERATE_PLOT_TEMPLATE_PREFIX + DATASET + GENERATE_PLOT_TEMPLATE_SUFIX
+    prompt = PromptTemplate(template=template, input_variables=['question'])
+    llm_chain = LLMChain(prompt=prompt, llm=llm1)
+    response = llm_chain.predict(question=question)
+    return response
+
+
+
+def retry_generate_plot(question, error_message, error_code):
+
+    dataset_first_3_rows = read_first_3_rows()
+    RETRY_TEMPLATE_PREFIX = """You are a high skilled visualization assistant that can modify a provided visualization code based on a set of instructions. You MUST return a full program. DO NOT include any preamble text. Do not include explanations or prose.
+    Current code attempts to create a visualization of dataset.csv to meet the objective. but it has encounted the given error. provide a corrected code. if you are adding comments or explanations they should start with #.
+
+#Example:
+import altair as alt
+import pandas as pd
+import streamlit as st
+
+# Read the dataset
+df = pd.read_csv('dataset.csv')
+
+# Calculate the total social media followers for each region
+region_followers = df.groupby('Region of Focus')[['X (Twitter) Follower #', 'Facebook Follower #', 'Instagram Follower #', 'Threads Follower #', 'YouTube Subscriber #', 'TikTok Subscriber #']].sum().reset_index()
+
+# Melt the dataframe to convert it into long format
+region_followers = region_followers.melt(id_vars='Region of Focus', var_name='Social Media', value_name='Total Followers')
+
+# Create the chart
+chart = alt.Chart(region_followers).mark_bar().encode(
+    x=alt.X('Region of Focus:N', title='Region of Focus'),
+    y=alt.Y('Total Followers:Q', title='Total Followers'),
+    color=alt.Color('Social Media:N', title='Social Media')
+)
+
+# Display the chart
+st.altair_chart(chart, use_container_width=True)    
+
+First 3 rows of the dataset:"""
+    DATASET = f"{dataset_first_3_rows}"
+
+
+    RETRY_TEMPLATE_SUFIX = """
+Objective: {question}
+
+Current Code:
+{error_code}
+
+Error Message:
+{error_message}
+
+Corrected Code:
+"""
+
+    retry_template = RETRY_TEMPLATE_PREFIX + DATASET + RETRY_TEMPLATE_SUFIX
+    retry_prompt = PromptTemplate(template=retry_template, input_variables=["question", "error_message, error_code"])
+
+    llm_chain = LLMChain(prompt=retry_prompt, llm=llm1)
+    response = llm_chain.predict(question=question, error_message=error_message, error_code=error_code)
+    return response

DV-AGENT/markup.py

@@ -0,0 +1,37 @@
+def app_intro():
+    return """
+    <div style='text-align: left; padding: 20px; border-radius: 10px;'>
+    <h1 style='text-align: center; color: #333;'>Student Performance Query System</h1>
+    <h2 style='text-align: center; color: #666;'>Query Student Performance Data with NLP</h2>
+
+    <p style='font-size: 18px; color: #444;'>Welcome to the Student Performance Query System! This application enables you to query student performance data using natural language processing (NLP) techniques. Originally designed with OpenAI functions, it has now been fully converted to utilize Mistral-8x7B-Instruct.</p>
+
+    <h3 style='color: #737373;'>Key Features: </h3>
+    <ul>
+        <li>Translate natural language queries into actionable insights! ✨</li>
+        <li>Engage in conversational queries to explore student performance data.</li>
+        <li>Powered by Mistral-8x7B-Instruct.</li>
+    </ul>
+
+    <h3 style='color: #737373;'>In the Black Box: ⚙️</h3>
+    <p style='font-size: 16px;'>This application utilizes the <a href="https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1">Mistral-8x7B-Instruct-v0.1 LLM</a> to interpret your queries and provide insights into student performance data.</p>
+
+    <h3 style='color: #737373;'>Get Started: </h3>
+    <p style='font-size: 16px;'>Ask questions about student performance in plain language and let the system handle the rest! 🎓 The bot is here to assist you. Dataset used: <a href="https://archive.ics.uci.edu/dataset/320/student+performance">Student Performance Data</a>.</p>
+    </div>
+    """
+
+
+def how_use_intro():
+    return """
+    <div style='text-align: left; padding: 20px; border-radius: 10px;'>
+    <h2 style='text-align: center; color: #333;'>Getting Started with Student Performance Query System! 📊🔍</h2>
+    <br>
+    <h3 style='color: #777;'>How to Use:</h3>
+    <ul style='font-size: 16px; color: #555;'>
+        <li><b>Query Student Performance:</b> Enter your questions about student performance in the provided input box using natural language. Click "Submit" to get answers and insights.</li>
+        <li><b>Interact with the Chatbot:</b> Engage in conversational queries with the chatbot. Ask about specific students, subjects, or trends to explore the dataset interactively.</li>
+    </ul>
+    <br>
+    </div>
+    """

DV-AGENT/memory.py

@@ -0,0 +1,5 @@
+from langchain.memory import ConversationBufferWindowMemory
+from langchain_community.chat_message_histories import StreamlitChatMessageHistory
+
+memory_storage = StreamlitChatMessageHistory(key="chat_messages")
+memory = ConversationBufferWindowMemory(memory_key="chat_history", human_prefix="User", chat_memory=memory_storage, k=3)

DV-AGENT/modules.py

@@ -0,0 +1,29 @@
+import streamlit as st
+import pandas as pd
+import os
+import shutil
+
+def replace_default_dataset():
+    # Replace dataset.csv with dataset_backup.csv
+    dataset_backup_path = "Student_copy.csv"
+    dataset_path = "Student.csv"
+
+    if os.path.exists(dataset_backup_path):
+        shutil.copy(dataset_backup_path, dataset_path)
+        st.success("Default dataset applied successfully.")
+    else:
+        st.warning("Default dataset backup not found.")
+
+def save_uploaded_dataset(uploaded_file):
+    # Save the uploaded dataset file and replace dataset.csv
+    dataset_path = "Student.csv"
+    uploaded_file.seek(0)
+
+    try:
+        df = pd.read_csv(uploaded_file)
+        df.to_csv(dataset_path, index=False)
+        st.success("Dataset uploaded and applied successfully.")
+    except pd.errors.EmptyDataError:
+        st.warning("Uploaded dataset is empty.")
+    except Exception as e:
+        st.error(f"An error occurred: {e}")

DV-AGENT/requirements.txt

@@ -0,0 +1,11 @@
+langchain
+langchain_experimental
+matplotlib
+altair
+streamlit
+streamlit_option_menu
+plotly
+tabulate
+langchain_community
+langchain-groq
+python-dotenv

DV-AGENT/tools.py

@@ -0,0 +1,111 @@
+from langchain_experimental.agents.agent_toolkits import create_csv_agent
+from langchain.agents.agent_types import AgentType
+from langchain.agents import Tool
+from langchain.chains import LLMMathChain
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import os
+import streamlit as st
+#from langchain_community.llms import HuggingFaceHub
+from typing import List
+from langchain_groq import ChatGroq
+from dotenv import load_dotenv
+load_dotenv()
+
+groq_api_key = os.getenv("GROQ_API_KEY")
+
+llm1 = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
+
+def csv_agnet(string):
+    agent = create_csv_agent(
+        llm1,
+        "dataset.csv",
+        verbose=True,
+        agent_type=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+    )
+
+    ans = agent.invoke(string)
+    return ans
+
+#def csv_agnet(string):
+#    agent = create_csv_agent(
+#        ChatOpenAI(temperature=0, model="gpt-3.5-turbo-0613"),
+#        "dataset.csv",
+#        verbose=True,
+#        agent_type=AgentType.OPENAI_FUNCTIONS,
+#    )
+
+#    ans = agent.run(string)
+#    return ans
+
+def math_tool(string):
+    #llm = OpenAI(temperature=0)
+    llm = llm1
+    llm_math_chain = LLMMathChain(llm=llm, verbose=True)
+    res = llm_math_chain.run(string)
+    return res
+
+def load_data():
+    df = pd.read_csv("dataset.csv", encoding="utf-8")
+    return df
+
+def plot_visualization(selected_option, x_column, y_column):
+    df = load_data()
+
+    if df.empty:
+        return st.warning("The data is empty.")
+
+    if x_column not in df.columns or y_column not in df.columns:
+        return st.warning("Invalid columns selected.")
+
+    if selected_option == "bar":
+        fig = px.bar(df, x=x_column, y=y_column, title=f"{x_column} vs {y_column}")
+    elif selected_option == "scatter":
+        fig = px.scatter(df, x=x_column, y=y_column, title=f"{x_column} vs {y_column}")
+    elif selected_option == "line":
+        fig = px.line(df, x=x_column, y=y_column, title=f"{x_column} vs {y_column}")
+    elif selected_option == "scatter_matrix":
+        fig = px.scatter_matrix(df, dimensions=[x_column, y_column], title=f"Scatter Matrix: {x_column} vs {y_column}")
+    elif selected_option == "box":
+        fig = px.box(df, x=x_column, y=y_column, title=f"Box Plot: {x_column} vs {y_column}")
+    elif selected_option == "heatmap":
+        fig = px.imshow(df.pivot_table(index=x_column, columns=y_column, aggfunc='size').fillna(0),
+                        labels=dict(x=x_column, y=y_column),
+                        title=f"Heatmap: {x_column} vs {y_column}")
+    else:
+        return st.warning("Please select a valid plot type.")
+
+    return st.plotly_chart(fig)
+
+
+def parsing_input(string):
+    selected_option, x_column, y_column = string.split(",")
+    return plot_visualization(selected_option, x_column, y_column)
+
+
+zeroshot_tools = [
+    Tool(
+        name="answer_qa",
+        func=csv_agnet,
+        description="Use this tool to query the dataset. input to this tool should be a standalone question. Include the correct row titles that are needed. Example Input format: How many rows are there in the dataset, which name has the highest calories",
+        #return_direct=True,
+    ),
+    Tool(
+        name="create_simple_plot",
+        func=parsing_input,
+        description="""Use this tool if the user asks to create x vs y plots. input must be a comma seperated list of: selected_option, x_column, y_column
+        Example Inputs: 
+        bar,calories,name
+
+        Allowed options are: bar, line, scatter_matrix, box, heatmap
+        you can decide plot type, x colllumn and y collumn based on the user input.
+        """,
+        #return_direct=True,
+    ),
+    Tool(
+        name="Calculator",
+        func=math_tool,
+        description="useful when you need to do calculations. Example input: 21^0.43"
+    ),
+]

DV-AGENT/v2/.env

@@ -0,0 +1 @@
+GROQ_API_KEY="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"

DV-AGENT/v2/app.py

@@ -0,0 +1,146 @@
+from langchain_groq import ChatGroq
+from langchain_experimental.agents import create_pandas_dataframe_agent
+import pandas as pd
+import json
+import streamlit as st
+from dotenv import load_dotenv
+import os
+load_dotenv()
+
+groq_api_key = os.getenv("GROQ_API_KEY")
+chat_model = ChatGroq(temperature=0, model_name="mixtral-8x7b-32768")
+
+def csv_tool(filename : str):
+
+    df = pd.read_csv(filename)
+    return create_pandas_dataframe_agent(chat_model,df, verbose=True)
+
+def ask_agent(agent, query):
+    """
+    Query an agent and return the response as a string.
+
+    Args:
+        agent: The agent to query.
+        query: The query to ask the agent.
+
+    Returns:
+        The response from the agent as a string.
+    """
+    # Prepare the prompt with query guidelines and formatting
+    prompt = (
+        """
+        Let's decode the way to respond to the queries. The responses depend on the type of information requested in the query. 
+
+        1. If the query requires a table, format your answer like this:
+           {"table": {"columns": ["column1", "column2", ...], "data": [[value1, value2, ...], [value1, value2, ...], ...]}}
+
+        2. For a bar chart, respond like this:
+           {"bar": {"columns": ["A", "B", "C", ...], "data": [25, 24, 10, ...]}}
+
+        3. If a line chart is more appropriate, your reply should look like this:
+           {"line": {"columns": ["A", "B", "C", ...], "data": [25, 24, 10, ...]}}
+
+        Note: We only accommodate two types of charts: "bar" and "line".
+
+        4. For a plain question that doesn't need a chart or table, your response should be:
+           {"answer": "Your answer goes here"}
+
+        For example:
+           {"answer": "The Product with the highest Orders is '15143Exfo'"}
+
+        5. If the answer is not known or available, respond with:
+           {"answer": "I do not know."}
+
+        Return all output as a string. Remember to encase all strings in the "columns" list and data list in double quotes. 
+        For example: {"columns": ["Products", "Orders"], "data": [["51993Masc", 191], ["49631Foun", 152]]}
+
+        Now, let's tackle the query step by step. Here's the query for you to work on: 
+        """
+        + query
+    )
+
+    # Run the prompt through the agent and capture the response.
+    response = agent.run(prompt)
+
+    # Return the response converted to a string.
+    return str(response)
+
+def decode_response(response: str) -> dict:
+    """This function converts the string response from the model to a dictionary object.
+
+    Args:
+        response (str): response from the model
+
+    Returns:
+        dict: dictionary with response data
+    """
+    return json.loads(response)
+
+def write_answer(response_dict: dict):
+    """app
+    Write a response from an agent to a Streamlit .
+
+    Args:
+        response_dict: The response from the agent.
+
+    Returns:
+        None.
+    """
+
+    # Check if the response is an answer.
+    if "answer" in response_dict:
+        st.write(response_dict["answer"])
+
+    # Check if the response is a bar chart.
+    # Check if the response is a bar chart.
+    if "bar" in response_dict:
+        data = response_dict["bar"]
+        try:
+            df_data = {
+                    col: [x[i] if isinstance(x, list) else x for x in data['data']]
+                    for i, col in enumerate(data['columns'])
+                }       
+            df = pd.DataFrame(df_data)
+            df.set_index("Products", inplace=True)
+            st.bar_chart(df)
+        except ValueError:
+            print(f"Couldn't create DataFrame from data: {data}")
+
+# Check if the response is a line chart.
+    if "line" in response_dict:
+        data = response_dict["line"]
+        try:
+            df_data = {col: [x[i] for x in data['data']] for i, col in enumerate(data['columns'])}
+            df = pd.DataFrame(df_data)
+            df.set_index("Products", inplace=True)
+            st.line_chart(df)
+        except ValueError:
+            print(f"Couldn't create DataFrame from data: {data}")
+
+
+    # Check if the response is a table.
+    if "table" in response_dict:
+        data = response_dict["table"]
+        df = pd.DataFrame(data["data"], columns=data["columns"])
+        st.table(df)
+st.set_page_config(page_title="👨‍💻 Talk with your CSV")
+st.title("👨‍💻 Talk with your CSV")
+
+st.write("Please upload your CSV file below.")
+
+data = st.file_uploader("Upload a CSV" , type="csv")
+
+query = st.text_area("Send a Message")
+
+if st.button("Submit Query", type="primary"):
+    # Create an agent from the CSV file.
+    agent = csv_tool(data)
+
+    # Query the agent.
+    response = ask_agent(agent=agent, query=query)
+
+    # Decode the response.
+    decoded_response = decode_response(response)
+
+    # Write the response to the Streamlit app.
+    write_answer(decoded_response)