Auto update

Dockerfile

@@ -8,7 +8,7 @@ COPY --chown=user . $HOME/app
 RUN chown -R user:user $HOME/app
 RUN chmod -R 755 $HOME/app
 COPY ./requirements.txt ~/app/requirements.txt
-RUN pip install chainlit openai modal
+RUN pip install chainlit openai modal search-engines
 RUN python3 -m modal setup
 COPY . .
 CMD ["chainlit", "run", "app.py", "--port", "7860"]

app.py

@@ -4,8 +4,10 @@ import base64
 import hashlib
 from datetime import datetime
 import os
+from ast import literal_eval
 import json
 import modal
+from browser import browse
 
 model = "gpt-4-turbo"
 client = OpenAI()
@@ -15,30 +17,31 @@ with open("system_prompt.txt", "r") as file:
    system_prompt = file.read()
    system_prompt.format(date = datetime.now().strftime("%Y-%m-%d"))
 
+#{
+#   "type": "function",
+#   "function": {
+#         "name": "python",
+#         "description": "Run Python code in a sandboxed environment and return the output.",
+#         "parameters": {
+#            "type": "object",
+#            "properties": {
+#               "command": {
+#                  "type": "string",
+#                  "description": "Optional. Command to run before the code. e.g. 'pip install numpy'"
+#               },
+#               "code": {
+#                  "type": "string",
+#                  "description": "The Python code to run."
+#               }
+#            },
+#            "required": [
+#               "code"
+#            ],
+#         },
+#   },
+#},
+
 tools = [
-   {
-      "type": "function",
-      "function": {
-            "name": "python",
-            "description": "Run Python code in a sandboxed environment and return the output.",
-            "parameters": {
-               "type": "object",
-               "properties": {
-                  "command": {
-                     "type": "string",
-                     "description": "Optional. Command to run before the code. e.g. 'pip install numpy'"
-                  },
-                  "code": {
-                     "type": "string",
-                     "description": "The Python code to run."
-                  }
-               },
-               "required": [
-                  "code"
-               ],
-            },
-      },
-   },
    {
       "type": "function",
       "function": {
@@ -61,6 +64,25 @@ tools = [
                ],
             },
       },
+   },
+   {
+      "type": "function",
+      "function": {
+         "name": "browse",
+         "description": "Browse the web to look for a given information. You only have to provide the information you are looking for and a system of advanced AI agents (which you can refer to as 'The Agent Swarm') will do the rest.",
+         "parameters": {
+            "type": "object",
+            "properties": {
+               "information": {
+                  "type": "string",
+                  "description": "A description of the information you are looking for. e.g 'When will the next SpaceX launch take place ?'"
+               }
+            },
+            "required": [
+               "information"
+            ],
+         },
+      }
    }
 ]
 
@@ -130,6 +152,13 @@ def handle_vision_call(msg, image_history):
       user_message = {"role": "user", "content": [{"type": "text", "text": msg.content}]}
       for image in image_base64:
          user_message["content"].append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image}", "detail": "high"}})
+      files = [file for file in msg.elements if "image" not in file.mime]
+      file_string = " <uploaded-files>"
+      for file in files:
+         file_string += f"'{file.path}', "
+      if file_string != " <uploaded-files>":
+         file_string += "</uploaded-files>"
+      user_message["content"][0]["text"] += file_string
       image_history.append(user_message)
       stream = gpt_vision_call(image_history)
       return stream
@@ -139,7 +168,6 @@ def gpt_vision_call(image_history: list = []):
       model=model,
       messages=image_history,
       tools = tools,
-      max_tokens=4096,
       temperature=0,
       stream=True
    )
@@ -157,10 +185,20 @@ async def on_message(msg: cl.Message):
    stream_msg = cl.Message(content="") 
    stream = None
 
-   if msg.elements:
+   if any("image" in file.mime for file in msg.elements): # if the message contains an image
       stream = handle_vision_call(msg, image_history)
    else:
-      image_history.append({"role": "user", "content": msg.content})
+      print("no images detected")
+      files = [file for file in msg.elements if "image" not in file.mime]
+      content = msg.content
+      if files:
+         print("files detected")
+         file_string = " <uploaded-files>"
+         for file in files:
+            file_string += f"'{file.path}', "
+         file_string += "</uploaded-files>"
+         content = msg.content + file_string
+      image_history.append({"role": "user", "content": content})
       stream = gpt_vision_call(image_history)
 
    if stream:
@@ -170,7 +208,15 @@ async def on_message(msg: cl.Message):
          image_history.append({"role": "assistant", "content": stream_infos.get("assistant_content", ""), "tool_calls": tool_calls_list})
          print(stream_infos.get("tool_calls"))
          for tool_call_id, tool_call in stream_infos.get("tool_calls").items():
-            arguments_json = json.loads(tool_call["arguments"])
+            try:
+               print(tool_call["arguments"])
+               arguments_json = json.loads(tool_call["arguments"])
+            except json.JSONDecodeError as e:
+               print(f"Error decoding JSON: {str(e)}")
+               try:
+                  arguments_json = eval(tool_call["arguments"])
+               except Exception as e:
+                  arguments_json = literal_eval(tool_call["arguments"])
             if tool_call["name"] == "dalle":
                function_response = generate_image(arguments_json["prompt"], arguments_json.get("size", "1024x1024"))
                if function_response["status"] == "success":
@@ -186,7 +232,19 @@ async def on_message(msg: cl.Message):
                   }
                )
             elif tool_call["name"] == "python":
+               print("Starting sandbox...")
                function_response = remote_execution.remote(command = arguments_json.get("command", None), code = arguments_json["code"])
+               print("Code executed!")
+               image_history.append(
+                  {
+                     "tool_call_id": tool_call_id,
+                     "role": "tool",
+                     "name": tool_call["name"],
+                     "content": str(function_response)
+                  }
+               )
+            elif tool_call["name"] == "browse":
+               function_response = browse(arguments_json["information"])
                image_history.append(
                   {
                      "tool_call_id": tool_call_id,

browser.py

@@ -0,0 +1,108 @@
+import modal
+import json
+from openai import OpenAI
+import time
+from datetime import datetime
+from tavily import TavilyClient
+import os
+
+tavily = TavilyClient(api_key=os.environ.get("TAVILY_API_KEY"))
+
+client = OpenAI()
+web_scraper = modal.Function.lookup("web-scraper", "extract_text")
+day = datetime.now().day
+month = datetime.now().strftime("%B")
+year = datetime.now().year
+print(f"We are the {day} of {month} {year}.")
+messages = [{"role": "system", "content": "You are in charge of browsing the web for information. To reach your goal, you can use 2 differents functions: `google_search` and `scrape`. Always start with a google search except if you already know the URL of the information you are looking for. Next, you can use the `scrape` function to extract the text from any page given its url. You can repeat this process as many times as you want. When you found the information you were looking for, you can use the `return` function to send it back to the user. We are the {day} of {month} {year}."}]
+functions = [
+   {
+      "name": "google_search",
+      "description": "Search for information on Google. Returns the URL of the 10 first results.",
+      "parameters": {
+         "type": "object",
+         "properties": {
+            "query": {
+               "type": "string",
+               "description": "The query to search for."
+            }
+         },
+         "required": [
+            "query"
+         ],
+      },
+   },
+   {
+      "name": "scrape",
+      "description": "Extract text from a web page given its URL. This will return all the text on the page.",
+      "parameters": {
+         "type": "object",
+         "properties": {
+            "url": {
+               "type": "string",
+               "description": "The URL of the page to scrape."
+            }
+         },
+         "required": [
+            "url"
+         ],
+      },
+   },
+   {
+      "name": "return",
+      "description": "Return a report of the search to the user.",
+      "parameters": {
+         "type": "object",
+         "properties": {
+            "response": {
+               "type": "string",
+               "description": "The report to send back to the user."
+            }
+         },
+         "required": [
+            "response"
+         ],
+      },
+   }
+]
+
+def google_search(query):
+   response = tavily.search(query=query, search_depth="advanced")
+   print(response)
+   context = [{"url": obj["url"], "content": obj["content"]} for obj in response["results"]]
+   return context
+
+def scrape(url):
+   return web_scraper.remote(url)
+
+def browse(information: str):
+   start = time.time()
+   messages.append({"role": "user", "content": f"Here is the information you need to find: '{information}'"})
+   while True:
+      print("Calling Agent Swarm...")
+      
+      action = client.chat.completions.create(
+         model = "gpt-4-turbo",
+         messages = messages,
+         temperature = 0,
+         functions = functions
+      ).choices[0].message
+      messages.append({"role": "assistant", "content": action.content, "function_call": {"name": action.function_call.name, "arguments": action.function_call.arguments} if action.function_call is not None else None})
+      if action.content is not None and action.function_call is None:
+         print(action)
+         print("Assistant reponded with content instead of a function call.")
+         messages.append({"role": "user", "content": "Please use a function to continue."})
+      elif action.function_call is not None:
+         print(f"Function: {action.function_call}")
+         function_name = action.function_call.name
+         arguments = json.loads(action.function_call.arguments)
+         if function_name == "return":
+            print("Returning information to user...")
+            print(arguments)
+            print(f"Time elapsed: {time.time() - start} seconds")
+            return {"status": "success", "response": arguments["response"]}
+         if function_name not in globals() or not callable(globals()[function_name]):
+            raise Exception(f"Function '{function_name}' not found.")
+         function_response = eval(f"{function_name}(**{arguments})")
+         print(f"Function response: {function_response}")
+         messages.append({"role": "function", "content": str(function_response), "name": function_name})

modal_scraper.py

@@ -0,0 +1,22 @@
+import modal
+from playwright.async_api import async_playwright
+
+playwright_image = modal.Image.debian_slim(python_version="3.10").run_commands(
+   "pip install playwright",
+   "playwright install-deps",
+   "playwright install"
+)
+app = modal.App(name="web-scraper", image=playwright_image)
+
+@app.function()
+async def extract_text(url, wait_for = None):
+   print('Extracting all text...', end="")
+   async with async_playwright() as p:
+      browser = await p.chromium.launch(headless=True)
+      page = await browser.new_page()
+      await page.goto(url)
+      if wait_for:
+         await page.wait_for_selector(wait_for)
+      text = await page.inner_text('body')
+      print(' Done')
+      return text

system_prompt.txt

@@ -5,4 +5,10 @@ Current date: {date}
 
 Image input capabilities: Enabled
 
+Important terms you can use:
+'The Creator': You can use it to refer to Octave
+'The Agent Swarm': You can use it to refer to the browsing AI agents that are in charge of looking for information on the web when you need it.
+
+Sometimes, the user will upload files that you can use with the python tool. You will be provided with the relative path of these files.
+
 Always answer in the user's language.