initial commit

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,28 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.DS_Store

README.md

@@ -1,2 +1,13 @@
-# AutoAgents
-AutoAgents!
+---
+title: Search Llm
+emoji: 🐢
+colorFrom: green
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.21.0
+python_version: 3.10.11
+app_file: autoagents/spaces/app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

autoagents/agents/search.py

@@ -0,0 +1,259 @@
+from typing import List, Union, Any, Optional, Dict
+import uuid
+import re
+from datetime import date
+import asyncio
+from collections import defaultdict
+
+from langchain.agents import Tool, AgentExecutor, LLMSingleActionAgent, AgentOutputParser
+from langchain.prompts import StringPromptTemplate
+from langchain import LLMChain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema import AgentAction, AgentFinish
+from langchain.callbacks import get_openai_callback
+from langchain.callbacks.base import AsyncCallbackHandler
+from langchain.callbacks.manager import AsyncCallbackManager
+
+
+from autoagents.tools.tools import search_tool, note_tool, rewrite_search_query
+from autoagents.utils.logger import InteractionsLogger
+
+
+# Set up the base template
+template = """
+We are working together to satisfy the user's original goal step-by-step. Play to your strengths as an LLM.
+Make sure the plan is achievable using the
+available tools. You SHOULD directly produce a `Final Answer:` when you
+think you have good-enough information to achieve the Goal. The final answer should be descriptive should be descriptive, encompassing all relevant details..
+Today is {today}.
+
+## Goal:
+{input}
+
+If you require assistance or additional information, you should use *only* one of the following tools:
+{tools}.
+
+## Output format
+You MUST produce Output in the following format:
+
+Thought: you should always think about what to do when you think you have not achieved the Goal.
+Reasoning: reasoning
+Plan:
+- short bulleted
+- list that conveys
+- next-step plan
+Action: the action to take, should be ONE OF {tool_names}
+Action Input: the input to the Action
+Observation: the result of the Action
+... (this Thought/Reasoning/Plan/Action/Action Input/Observation can repeat N
+times until there is a Final Answer)
+Final Answer: the final answer to achieve the original Goal which can be the
+only output or when you have no Action to do next.
+
+## History
+{agent_scratchpad}
+
+Do not repeat any past actions in History, because you will not get additional information.
+If the last action is search, then you should use notepad to keep critical information.
+If you have gathered all information in your plannings to satisfy the user's original goal, then respond immediately as the Final Answer.
+"""
+
+
+# Set up a prompt template
+class CustomPromptTemplate(StringPromptTemplate):
+    # The template to use
+    template: str
+    # The list of tools available
+    tools: List[Tool]
+    ialogger: InteractionsLogger
+
+    def format(self, **kwargs) -> str:
+        # Get the intermediate steps (AgentAction, Observation tuples)
+        # Format them in a particular way
+        intermediate_steps = kwargs.pop("intermediate_steps")
+        outputs = ""
+        # Set the agent_scratchpad variable to that value
+        for action, observation in intermediate_steps[:-1]:
+            outputs += f"{action.log}\n"
+        if len(intermediate_steps) > 0:
+            action, observation = intermediate_steps[-1]
+            self.ialogger.add_system({"action": action, "observation": observation})
+            if action.tool not in ("Search", "Notepad"):
+                raise Exception("Invalid tool requested by the model.")
+            if action.tool == "Notepad":
+                outputs += f"{action.log}\n"
+                outputs += f"Observation: {observation}\n"
+            elif action.tool == "Search":
+                current = "".join([f"{d}" for d in observation])
+                outputs += f"{action.log}\n"
+                outputs += f"Observation: {current}\n"
+
+            # Parse the output ofr the last step for the reasoning and plan
+            regex = r"Thought\s*\d*\s*:(.*?)\n(.*)"
+            match = re.search(regex, action.log, re.DOTALL)
+            thoughts = match.group(1).strip() if match else ""
+
+            regex = r"Reasoning\s*\d*\s*:(.*?)\n(.*)"
+            match = re.search(regex, action.log, re.DOTALL)
+            reasoning = match.group(1).strip() if match else ""
+
+            regex = r"Plan\s*\d*\s*:(.*?)\nAction(.*)"
+            match = re.search(regex, action.log, re.DOTALL)
+            plans = match.group(1).strip() if match else ""
+            self.ialogger.add_structured_data({"output":{"thoughts": thoughts,
+                                                         "reasoning": reasoning,
+                                                         "plans": plans,
+                                                         "action": action.tool,
+                                                         "action_input": action.tool_input,
+                                                         "raw_output":action.log},
+                                                         "observation": observation})
+        kwargs["agent_scratchpad"] = outputs
+        # Create a tools variable from the list of tools provided
+        kwargs["tools"] = "\n".join([f"{tool.name}: {tool.description}" for tool in self.tools])
+        # Create a list of tool names for the tools provided
+        kwargs["tool_names"] = ", ".join([tool.name for tool in self.tools])
+        kwargs["today"] = date.today()
+        final_prompt = self.template.format(**kwargs)
+        if not intermediate_steps:
+            # first iteration
+            self.ialogger.add_system({"prompt": final_prompt})
+        return final_prompt
+
+
+class CustomOutputParser(AgentOutputParser):
+    class Config:
+        arbitrary_types_allowed = True
+    ialogger: InteractionsLogger
+    api_key: str
+    new_action_input: Optional[str]
+
+    action_history = defaultdict(set)
+
+    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
+        self.ialogger.add_ai(llm_output)
+        # Check if agent should finish
+        if "Final Answer:" in llm_output:
+            final_answer = llm_output.split("Final Answer:")[-1].strip()
+            self.ialogger.add_structured_data({"output": {"action": "Final Answer",
+                                                          "action_input": final_answer,
+                                                          "raw_output": llm_output}})
+            return AgentFinish(
+                # Return values is generally always a dictionary with a single `output` key
+                # It is not recommended to try anything else at the moment :)
+                return_values={"output": final_answer},
+                log=llm_output,
+            )
+        # Parse out the action and action input
+        regex = r"Action\s*\d*\s*:(.*?)\nAction\s*\d*\s*Input\s*\d*\s*:[\s]*(.*)"
+        match = re.search(regex, llm_output, re.DOTALL)
+        if not match:
+            raise ValueError(f"Could not parse LLM output: `{llm_output}`")
+        action = match.group(1).strip()
+        action_input = match.group(2).strip().strip('"')
+
+        if action_input in self.action_history[action]:
+            new_action_input = rewrite_search_query(action_input,
+                                                    self.action_history[action],
+                                                    self.api_key)
+            self.new_action_input = new_action_input
+            self.action_history[action].add(new_action_input)
+            return AgentAction(tool=action, tool_input=new_action_input, log=llm_output)
+        else:
+            # Return the action and action input
+            self.action_history[action].add(action_input)
+            return AgentAction(tool=action, tool_input=action_input, log=llm_output)
+
+
+class ActionRunner:
+    def __init__(self, outputq, api_key: str, model_name: str, persist_logs=False):
+        self.ialogger = InteractionsLogger(name=f"{uuid.uuid4().hex[:6]}", persist=persist_logs)
+        tools = [search_tool, note_tool]
+        prompt = CustomPromptTemplate(
+                template=template,
+                tools=tools,
+                input_variables=["input", "intermediate_steps"],
+                ialogger=self.ialogger)
+
+        output_parser = CustomOutputParser(ialogger=self.ialogger, api_key=api_key)
+
+        class MyCustomHandler(AsyncCallbackHandler):
+            def __init__(self):
+                pass
+
+            async def on_chain_end(self, outputs, **kwargs) -> None:
+                if "text" in outputs:
+                    await outputq.put(outputs["text"])
+
+            async def on_agent_action(
+                    self,
+                    action: AgentAction,
+                    *,
+                    run_id: uuid.UUID,
+                    parent_run_id: Optional[uuid.UUID] = None,
+                    **kwargs: Any,
+                    ) -> None:
+                if (new_action_input := output_parser.new_action_input):
+                    # Notify users
+                    await outputq.put(RuntimeWarning(f"Action Input Rewritten: {new_action_input}"))
+                    output_parser.new_action_input = None
+
+            async def on_tool_start(
+                    self,
+                    serialized: Dict[str, Any],
+                    input_str: str,
+                    *,
+                    run_id: uuid.UUID,
+                    parent_run_id: Optional[uuid.UUID] = None,
+                    **kwargs: Any,
+                    ) -> None:
+                pass
+
+            async def on_tool_end(
+                    self,
+                    output: str,
+                    *,
+                    run_id: uuid.UUID,
+                    parent_run_id: Optional[uuid.UUID] = None,
+                    **kwargs: Any,
+                    ) -> None:
+                await outputq.put(output)
+
+        handler = MyCustomHandler()
+
+        llm = ChatOpenAI(openai_api_key=api_key, temperature=0, model_name=model_name)
+        llm_chain = LLMChain(llm=llm, prompt=prompt, callbacks=[handler])
+        tool_names = [tool.name for tool in tools]
+        for tool in tools:
+            tool.callbacks = [handler]
+
+        agent = LLMSingleActionAgent(
+            llm_chain=llm_chain,
+            output_parser=output_parser,
+            stop=["\nObservation:"],
+            allowed_tools=tool_names
+        )
+        callback_manager = AsyncCallbackManager([handler])
+
+        # Finally create the Executor
+        self.agent_executor = AgentExecutor.from_agent_and_tools(agent=agent,
+                                                                 tools=tools,
+                                                                 verbose=False,
+                                                                 callback_manager=callback_manager)
+
+    async def run(self, goal: str, outputq):
+        self.ialogger.set_goal(goal)
+        try:
+            with get_openai_callback() as cb:
+                output = await self.agent_executor.arun(goal)
+                self.ialogger.add_cost({"total_tokens": cb.total_tokens,
+                                        "prompt_tokens": cb.prompt_tokens,
+                                        "completion_tokens": cb.completion_tokens,
+                                        "total_cost": cb.total_cost,
+                                        "successful_requests": cb.successful_requests})
+            self.ialogger.save()
+        except Exception as e:
+            self.ialogger.add_message({"error": str(e)})
+            self.ialogger.save()
+            await outputq.put(Exception("Something went wrong. Please try searching again."))
+            return
+        return output

autoagents/spaces/app.py

@@ -0,0 +1,139 @@
+import os
+import asyncio
+import random
+from ast import literal_eval
+
+import streamlit as st
+import openai
+
+from autoagents.utils.constants import MAIN_HEADER, MAIN_CAPTION, SAMPLE_QUESTIONS
+from autoagents.agents.search import ActionRunner
+
+
+async def run():
+    output_acc = ""
+    st.session_state["random"] = random.randint(0, 99)
+    if "task" not in st.session_state:
+        st.session_state.task = None
+    if "model_name" not in st.session_state:
+        st.session_state.model_name = "gpt-3.5-turbo"
+
+    st.set_page_config(
+        page_title="Search Agent",
+        page_icon="🤖",
+        layout="wide",
+        initial_sidebar_state="expanded",
+    )
+
+    st.title(MAIN_HEADER)
+    st.caption(MAIN_CAPTION)
+
+    with st.form("my_form", clear_on_submit=False):
+        st.markdown("<style> .inter { white-space: pre-line; } </style>", unsafe_allow_html=True)
+        user_input = st.text_input(
+            "You: ",
+            key="input",
+            placeholder="Ask me anything ...",
+            label_visibility="hidden",
+        )
+
+        submitted = st.form_submit_button(
+            "Search", help="Hit to submit the search query."
+        )
+
+        # Ask the user to enter their OpenAI API key
+        API_O = st.sidebar.text_input("OpenAI api-key", type="password") or os.getenv(
+            "OPENAI_API_KEY"
+        )
+        with st.sidebar:
+            model_dict = {
+                "gpt-3.5-turbo": "GPT-3.5-turbo",
+                "gpt-4": "GPT-4 (Recommneded for better quality results)",
+            }
+            st.radio(
+                "OpenAI model",
+                model_dict.keys(),
+                key="model_name",
+                format_func=lambda x: model_dict[x],
+            )
+
+            st.markdown("**Example Queries:**")
+            for q in SAMPLE_QUESTIONS:
+                st.markdown(f"*{q}*")
+
+        if not API_O:
+            st.warning(
+                "API key required to try this app. The API key is not stored in any form. [This](https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key) might help."
+            )
+        else:
+            outputq = asyncio.Queue()
+            runner = ActionRunner(
+                outputq,
+                api_key=API_O,
+                model_name=st.session_state.model_name,
+                persist_logs=True,
+            )  # log to HF-dataset
+
+            async def cleanup(log: str):
+                st.error(log)
+                await st.session_state.task
+                st.session_state.task = None
+                st.stop()
+
+            placeholder = st.empty()
+
+            if user_input and submitted:
+                if st.session_state.task is not None:
+                    with placeholder.container():
+                        st.session_state.task.cancel()
+                        st.warning("Previous search aborted", icon="⚠️")
+
+                st.session_state.task = asyncio.create_task(
+                    runner.run(user_input, outputq)
+                )
+                iterations = 0
+                with st.expander("Search Results", expanded=True):
+                    while True:
+                        with st.spinner("Wait for it..."):
+                            output = await outputq.get()
+                            placeholder.empty()
+                            if isinstance(output, Exception):
+                                if isinstance(output, openai.error.AuthenticationError):
+                                    await cleanup(
+                                        f"AuthenticationError: {output.json_body}"
+                                    )
+                                elif isinstance(output, RuntimeWarning):
+                                    st.warning(output)
+                                    continue
+                                else:
+                                    await cleanup(str(output))
+                                return
+                            try:
+                                output_fmt = literal_eval(output)
+                                st.json(output_fmt, expanded=False)
+                                st.write("---")
+                                iterations += 1
+                            except:
+                                output_acc += "\n" + output
+                                st.markdown(f"<div class=\"inter\"> {output} </div>",
+                                            unsafe_allow_html=True)
+                            if iterations >= runner.agent_executor.max_iterations:
+                                await cleanup(
+                                    f"Maximum iterations ({iterations}) exceeded. You can try running the search again or try a variation of the query."
+                                )
+                                return
+                            if "Final Answer:" in output:
+                                break
+                # Found the answer
+                final_answer = await st.session_state.task
+                final_answer = final_answer.replace("$", "\$")
+                # st.success accepts md
+                st.success(final_answer, icon="✅")
+                st.balloons()
+                st.session_state.task = None
+                st.stop()
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.set_debug(enabled=False)
+    loop.run_until_complete(run())

autoagents/tools/tools.py

@@ -0,0 +1,64 @@
+from langchain import PromptTemplate, OpenAI, LLMChain
+from langchain.agents import Tool
+from duckpy import Client
+from langchain.chat_models import ChatOpenAI
+
+
+MAX_SEARCH_RESULTS = 20  # Number of search results to observe at a time
+
+search_description = """ Useful for when you need to ask with search. Use direct language and be
+EXPLICIT in what you want to search.
+
+## Examples of incorrect use
+1.  Action: Search
+    Action Input: "[name of bagel shop] menu"
+
+The Action Input cannot be None or empty.
+"""
+
+notepad_description = """ Useful for when you need to note-down specific
+information for later reference. Please provide full information you want to
+note-down in the Action Input and all future prompts will remember it.
+This is the mandatory tool after using the search tool.
+Using Notepad does not always lead to a final answer.
+
+## Exampels of using notepad tool
+Action: Notepad
+Action Input: the information you want to note-down
+"""
+
+async def ddg(query: str):
+    if query is None or query.lower().strip().strip('"') == "none" or query.lower().strip().strip('"') == "null":
+        x = "The action input field is empty. Please provide a search query."
+        return [x]
+    else:
+        client = Client()
+        return client.search(query)[:MAX_SEARCH_RESULTS]
+
+
+async def notepad(x: str) -> str:
+    return f"{[x]}"
+
+
+search_tool = Tool(name="Search",
+                   func=lambda x: x,
+                   coroutine=ddg,
+                   description=search_description)
+
+note_tool = Tool(name="Notepad",
+                   func=lambda x: x,
+                   coroutine=notepad,
+                   description=notepad_description)
+
+
+def rewrite_search_query(q: str, search_history, api_key: str) -> str:
+    history_string = '\n'.join(search_history)
+    template ="""We are using the Search tool.
+                 # Previous queries:
+                 {history_string}. \n\n Rewrite query {action_input} to be
+                 different from the previous ones."""
+    llm = ChatOpenAI(temperature=0, openai_api_key=api_key)
+    prompt = PromptTemplate(template=template,
+                            input_variables=["action_input", "history_string"])
+    llm_chain = LLMChain(prompt=prompt, llm=llm)
+    return llm_chain.predict(action_input=q, history_string=history_string)

autoagents/utils/constants.py

@@ -0,0 +1,13 @@
+MAIN_HEADER = "Web Search Agent"
+
+MAIN_CAPTION = """ This is a proof-of-concept search engine built on ReAct-style
+prompting which acts as a search agent that plans and executes web searches on
+your behalf. Given a high-level search query the agent tries to come up with a
+concluding answer based off multiple rounds of searches. You can observe all
+the intermediate interactions with the search engine below."""
+
+SAMPLE_QUESTIONS = [
+    "What has David Sacks written about SAAS? Can you provide some links?",
+    "Where is the all-in summit 2023 being held and how much are the tickets?",
+    "Did Stan Druckenmiller buy Nvidia recently?",
+]

autoagents/utils/logger.py

@@ -0,0 +1,56 @@
+import os
+import json
+from typing import Dict, Any
+import uuid
+
+import huggingface_hub
+from huggingface_hub import Repository
+
+
+class InteractionsLogger:
+    def __init__(self, name: str, persist=False):
+        self.persist = persist
+        self.counter = 0
+        self.name = name  # unique id
+        HF_TOKEN = os.environ.get("HF_TOKEN")
+        HF_DATASET_REPO_URL = os.environ.get("HF_DATASET_REPO_URL")
+        if (HF_TOKEN is not None) and (HF_DATASET_REPO_URL is not None):
+            self.repo = Repository(
+                local_dir="data", clone_from=HF_DATASET_REPO_URL, use_auth_token=HF_TOKEN
+            )
+        else:
+            self.persist = False
+
+    def set_goal(self, goal: str):
+        # Initialize two variables for saving two files (self.messages for
+        # training and self.structure_data for later use)
+        self.messages = [{"goal": goal}]
+        self.structured_data = {"goal": goal}
+
+    def add_system(self, more: Dict):
+        self.convos = [{"from": "system"} | more]
+
+    def add_ai(self, msg: str):
+        self.convos.append({"from": "ai", "value": msg})
+        self.messages.append({"id": f"{self.name}_{self.counter}", "conversations": self.convos})
+        self.counter += 1
+
+    def add_structured_data(self, data: Dict[str, Any]):
+        self.structured_data.update({f"turn_{self.counter}": data})
+
+    def add_message(self, data: Dict[str, Any]):
+        self.structured_data.update(data)
+
+    def save(self):
+        if self.persist:
+            # TODO: want to add retry in a loop?
+            self.repo.git_pull()
+            fname = uuid.uuid4().hex[:16]
+            with open(f"./data/{fname}.json", "w") as f:
+                json.dump(self.messages, f, indent=2)
+            with open(f"./data/{fname}.clean.json", "w") as f:
+                json.dump(self.structured_data, f, indent=2)
+            commit_url = self.repo.push_to_hub()
+
+    def add_cost(self, cost):
+        self.messages.append({"metrics": cost})

requirements.txt

@@ -0,0 +1,4 @@
+openai>=0.27.7
+langchain
+duckpy
+huggingface_hub

test.py

@@ -0,0 +1,49 @@
+import os
+import asyncio
+from action import ActionRunner
+from langchain.callbacks import get_openai_callback
+from pprint import pprint
+import pdb
+from ast import literal_eval
+
+async def main(user_input):
+    outputq = asyncio.Queue()
+
+    API_O = os.getenv("OPENAI_API_KEY")
+    runner = ActionRunner(outputq, api_key=API_O, model_name="gpt-3.5-turbo")
+    task = asyncio.create_task(runner.run(user_input, outputq))
+
+    while True:
+        output = await outputq.get()
+        if isinstance(output, Exception):
+            print(output)
+            return
+        try:
+            pprint(literal_eval(output))
+        except:
+            print(output)
+        print("-----------------------------------------------------------")
+        if "Final Answer:" in output:
+            break
+    await task
+
+
+# "list 5 cities and their current populations where Paramore is playing this year.",
+# "Who is Leo DiCaprio's girlfriend? What is her current age raised to the 0.43 power?",
+# "How many watermelons can fit in a Tesla Model S?",
+# "Recommend me some laptops suitable for UI designers under $2000. Please include brand and price."
+Q = [
+     "Build me a vacation plan for Rome and Milan this summer for seven days. Include place to visit and hotels to stay. ",
+     "What is the sum of ages of the wives of Barack Obama and Donald Trump?",
+     "Who is the most recent NBA MVP? Which team does he play for? What is his season stats?",
+     "What were the scores for the last three games for the Los Angeles Lakers? Provide the dates and opposing teams.",
+     "Which team won in women's volleyball in the Summer Olympics that was held in London?",
+     "Provide a summary of the latest COVID-19 research paper published. Include the title, authors and abstract.",
+     "What is the top grossing movie in theatres this week? Provide the movie title, director, and a brief synopsis of the movie's plot. Attach a review for this movie.",
+     "Recommend a bagel shop near the Strip district in Pittsburgh that offer vegan food",
+     "Who are some top researchers in the field of machine learning systems nowadays?"
+     ]
+
+loop = asyncio.new_event_loop()
+for i in range(len(Q)):
+    loop.run_until_complete(main(Q[i]))