Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,167 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+orb.py
+orb_app.py
+dev
+prompts/orb.py
+screenshots
+temp
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env*
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,13 +1,28 @@
 ---
-title: Devon
-emoji: ⚡
-colorFrom: red
-colorTo: green
-sdk: gradio
-sdk_version: 4.28.3
+title: DevOn
 app_file: app.py
-pinned: false
-license: apache-2.0
+sdk: gradio
+sdk_version: 4.24.0
 ---
+# DevOn
+
+What if you tried to do what Devin does, but using MultiOn's agents?
+
+We use 3 separate MultiOn Agents, one each for programming, researching and notetaking.
+
+Their activities are orchestrated and supervised by an overarching GPT-4V
+
+Setup:
+
+- Create a .env.local and set the following env variables:
+  - OPENAI_API_KEY
+  - MULTION_API_KEY
+  - REPLIT_EMAIL (if using the remote API)
+  - REPLIT_PASSWORD (if using the remote API)
+- ```pip install -r requirements.txt```
+- If you want to use the local API (need to have the chrome extension installed and API enabled for this): ```bash start_local.sh```
+- If you want to use the remote API: ```bash start_remote.sh```
+
+Barebones Demo:
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+https://github.com/lordspline/DevOn/assets/74811063/a8bd9d65-42de-4bf1-a721-79f9b4d772ef

agent.py

@@ -0,0 +1,310 @@
+from openai import OpenAI
+from prompts.orchestrator import orchestrator_prompt
+from prompts.programmer import programmer_notes
+from prompts.notetaker import notetaker_notes
+from dotenv import load_dotenv
+import time
+import multion
+from multion.client import MultiOn
+import os
+
+load_dotenv(".env.local")
+
+replit_email = os.getenv("REPLIT_EMAIL")
+replit_password = os.getenv("REPLIT_PASSWORD")
+
+multion_api_key = os.getenv("MULTION_API_KEY")
+# multion.login(use_api=True, multion_api_key=multion_api_key)
+multion = MultiOn(api_key=multion_api_key)
+
+# runpod_url = os.getenv("RUNPOD_URL")
+
+image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg"
+
+
+class DevOn:
+    def __init__(self, editor_image, browser_image, scratchpad_image):
+        self.editor_image = editor_image
+        self.browser_image = browser_image
+        self.scratchpad_image = scratchpad_image
+        self.local = os.getenv("WHERE_EXECUTE") == "local"
+
+        self.programmer = multion.sessions.create(
+            url="https://replit.com/login", local=self.local, include_screenshot=True
+        )
+        self.programmer_logged_in = False
+        # self.editor_image = self.programmer.screenshot
+        self.editor_image = multion.sessions.screenshot(
+            session_id=self.programmer.session_id
+        ).screenshot
+        print(self.editor_image)
+        time.sleep(1)
+        # print(self.programmer)
+
+        self.researcher = multion.sessions.create(
+            url="https://www.google.com", local=self.local, include_screenshot=True
+        )
+        # self.browser_image = self.researcher.screenshot
+        self.browser_image = multion.sessions.screenshot(
+            session_id=self.researcher.session_id
+        ).screenshot
+        time.sleep(1)
+
+        self.notetaker = multion.sessions.create(
+            url="https://anotepad.com/", local=self.local, include_screenshot=True
+        )
+        # self.scratchpad_image = self.notetaker.screenshot
+        self.scratchpad_image = multion.sessions.screenshot(
+            session_id=self.notetaker.session_id
+        ).screenshot
+        time.sleep(1)
+
+        self.done = True
+        self.task = ""
+        self.plan = ""
+        self.messages = []
+        self.client = OpenAI()
+
+    def programmer_login(self):
+        if self.local:
+            cmd = "Create a new Python REPL."
+        else:
+            cmd = "Log in with the email {email} and the password {password}. Then create a new Python REPL.".format(
+                email=replit_email, password=replit_password
+            )
+        while True:
+            self.programmer = multion.sessions.step(
+                self.programmer.session_id,
+                cmd=cmd + "\n\n" + programmer_notes,
+                url="https://replit.com/login",
+                include_screenshot=True,
+            )
+            print(self.programmer)
+            print(
+                multion.sessions.screenshot(
+                    session_id=self.programmer.session_id
+                ).screenshot
+            )
+            # time.sleep(1)
+            # yield ("", self.editor_image, self.browser_image, self.scratchpad_image)
+            # self.editor_image = self.programmer["screenshot"]
+            if self.programmer.status == "DONE":
+                break
+
+        self.editor_image = multion.sessions.screenshot(
+            session_id=self.programmer.session_id
+        ).screenshot
+        time.sleep(1)
+
+    def prepare_messages(self):
+        messages = [
+            {"role": "user", "content": orchestrator_prompt},
+            {
+                "role": "user",
+                "content": "The Task given to you is: {task}".format(task=self.task),
+            },
+            {
+                "role": "user",
+                "content": "The current Plan state is: {plan}".format(plan=""),
+            },
+        ]
+        for message in self.messages:
+            messages.append(message)
+
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "This is the current state of the Programmer Intern.",
+                    },
+                    {"type": "image_url", "image_url": {"url": self.editor_image}},
+                ],
+            }
+        )
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "This is the current state of the Researcher Intern.",
+                    },
+                    {"type": "image_url", "image_url": {"url": self.browser_image}},
+                ],
+            }
+        )
+        messages.append(
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "This is the current state of the Notetaker Intern.",
+                    },
+                    {"type": "image_url", "image_url": {"url": self.scratchpad_image}},
+                ],
+            }
+        )
+        return messages
+
+    def execute_action(self, action):
+
+        action_func = action.split(" ", 1)[0]
+
+        if action_func == "submit":
+            self.done = True
+            return
+        elif action_func == "update_plan":
+            action_arg = action.split(" ", 1)[1]
+            self.plan = action_arg
+            return
+        elif action_func == "programmer":
+            action_arg = action.split(" ", 1)[1]
+            while True:
+                self.programmer = multion.sessions.step(
+                    self.programmer.session_id,
+                    cmd=action_arg + "\n\n" + programmer_notes,
+                    url="https://replit.com/login",
+                    include_screenshot=True,
+                )
+                print(self.programmer)
+                if self.programmer.status == "NOT SURE":
+                    self.messages.append(
+                        {
+                            "role": "user",
+                            "content": "The Programmer says: {message}\n\nYour next reply will go to the programmer.".format(
+                                message=self.programmer.message
+                            ),
+                        }
+                    )
+                    chat_completion = self.client.chat.completions.create(
+                        messages=self.prepare_messages(),
+                        model="gpt-4-vision-preview",
+                        # max_tokens=200,
+                    )
+                    action_arg = chat_completion.choices[0].message.content
+                    self.messages.append({"role": "assistant", "content": action_arg})
+                else:
+                    self.messages.append(
+                        {
+                            "role": "user",
+                            "content": "The Programmer says: {message}".format(
+                                message=self.programmer.message
+                            ),
+                        }
+                    )
+                # time.sleep(1)
+                # self.editor_image = self.programmer["screenshot"]
+                # yield ("", self.editor_image, self.browser_image, self.scratchpad_image)
+                if self.programmer.status == "DONE":
+                    break
+            self.editor_image = multion.sessions.screenshot(
+                session_id=self.programmer.session_id
+            ).screenshot
+            print(self.editor_image)
+            time.sleep(1)
+            return
+        elif action_func == "researcher":
+            action_arg = action.split(" ", 1)[1]
+            while True:
+                self.researcher = multion.sessions.step(
+                    self.researcher.session_id,
+                    cmd=action_arg,
+                    url="https://www.google.com",
+                    include_screenshot=True,
+                )
+                print(self.researcher)
+                self.messages.append(
+                    {
+                        "role": "user",
+                        "content": "The Researcher says: {message}".format(
+                            message=self.researcher.message
+                        ),
+                    }
+                )
+                # time.sleep(1)
+                # self.browser_image = self.researcher["screenshot"]
+                # yield ("", self.editor_image, self.browser_image, self.scratchpad_image)
+                if self.researcher.status == "DONE":
+                    break
+            self.browser_image = multion.sessions.screenshot(
+                session_id=self.researcher.session_id
+            ).screenshot
+            print(self.browser_image)
+            time.sleep(1)
+            return
+        elif action_func == "notetaker":
+            action_arg = action.split(" ", 1)[1]
+            while True:
+                self.notetaker = multion.sessions.step(
+                    self.notetaker.session_id,
+                    cmd=action_arg + "\n\n" + notetaker_notes,
+                    url="https://anotepad.com/",
+                    include_screenshot=True,
+                )
+                print(self.notetaker)
+                self.messages.append(
+                    {
+                        "role": "user",
+                        "content": "The Notetaker says: {message}".format(
+                            message=self.notetaker.message
+                        ),
+                    }
+                )
+                # time.sleep(1)
+                # self.scratchpad_image = self.notetaker["screenshot"]
+                # yield ("", self.editor_image, self.browser_image, self.scratchpad_image)
+                if self.notetaker.status == "DONE":
+                    break
+            self.scratchpad_image = multion.sessions.screenshot(
+                session_id=self.notetaker.session_id
+            ).screenshot
+            print(self.scratchpad_image)
+            time.sleep(1)
+            return
+        elif action_func == "clarify":
+            action_arg = action.split(" ", 1)[1]
+            return
+
+    def orchestrator(self):
+        if not self.programmer_logged_in:
+            self.programmer_login()
+            self.programmer_logged_in = True
+        messages = self.prepare_messages()
+        chat_completion = self.client.chat.completions.create(
+            messages=messages,
+            model="gpt-4-vision-preview",
+            # max_tokens=200,
+        )
+        response = chat_completion.choices[0].message.content
+        action, explanation = response.split("Explanation: ", 1)
+        action = action.split("Action: ", 1)[1]
+
+        self.messages.append({"role": "assistant", "content": response})
+        self.messages.append(
+            {
+                "role": "user",
+                "content": "The current Plan state is: {plan}".format(plan=self.plan),
+            }
+        )
+        print(self.messages)
+
+        self.execute_action(action)
+
+        # temp
+        # self.done = True
+        return explanation
+
+    def run(self, prompt):
+        self.done = False
+        self.task = prompt
+        while not self.done:
+            curr_response = self.orchestrator()
+            yield (
+                curr_response,
+                self.editor_image,
+                self.browser_image,
+                self.scratchpad_image,
+            )

app.py

@@ -0,0 +1,91 @@
+import gradio as gr
+import os
+import time
+from agent import DevOn
+
+image_temp = "https://miro.medium.com/v2/resize:fit:1200/0*n-2bW82Z6m6U2bij.jpeg"
+# devon = DevOn(
+#     editor_image=image_temp, browser_image=image_temp, scratchpad_image=image_temp
+# )
+devon = None
+
+
+def add_message(history, message):
+    for x in message["files"]:
+        history.append(((x,), None))
+    if message["text"] is not None:
+        history.append((message["text"], None))
+    return history, gr.MultimodalTextbox(value=None, interactive=False)
+
+
+def bot(history):
+    for r in devon.run(history[-1][0]):
+        text, editor_image, browser_image, scratchpad_image = r
+        if type(text) == str:
+            history.append((None, text))
+        if editor_image is None:
+            editor_image = devon.editor_image
+            browser_image = devon.browser_image
+            scratchpad_image = devon.scratchpad_image
+        yield history, editor_image, browser_image, scratchpad_image
+
+
+with gr.Blocks(css="footer {visibility: hidden}") as demo:
+    with gr.Row():
+        with gr.Column():
+            chatbot = gr.Chatbot(
+                [], elem_id="chatbot", bubble_full_width=False, height=300
+            )
+
+            chat_input = gr.MultimodalTextbox(
+                value={"text": "write a basic hello world fastapi server"},
+                interactive=True,
+                file_types=["image"],
+                placeholder="Enter message or upload file...",
+                show_label=False,
+            )
+        with gr.Column():
+            if devon:
+                editor_view = gr.Image(
+                    devon.editor_image,
+                    label="Editor",
+                )
+            else:
+                editor_view = gr.Image()
+    with gr.Row():
+        with gr.Column():
+            if devon:
+                browser_view = gr.Image(
+                    devon.browser_image,
+                    label="Browser",
+                )
+            else:
+                browser_view = gr.Image()
+        with gr.Column():
+            if devon:
+                scratchpad_view = gr.Image(
+                    devon.scratchpad_image,
+                    label="Scratchpad",
+                )
+            else:
+                scratchpad_view = gr.Image()
+
+    chat_msg = chat_input.submit(
+        add_message, [chatbot, chat_input], [chatbot, chat_input]
+    )
+    bot_msg = chat_msg.then(
+        bot,
+        [chatbot],
+        [chatbot, editor_view, browser_view, scratchpad_view],
+        api_name="bot_response",
+    )
+    bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
+
+    # chatbot.like(print_like_dislike, None, None)
+
+if __name__ == "__main__":
+    devon = DevOn(
+        editor_image=image_temp, browser_image=image_temp, scratchpad_image=image_temp
+    )
+    demo.queue()
+    demo.launch()

editor.py

@@ -0,0 +1,67 @@
+import gradio as gr
+import os
+import time
+import ngrok
+from dotenv import load_dotenv
+
+load_dotenv(".env.local")
+
+
+def new_file(file_name):
+    # print(file_explorer)
+    new_file_path = os.path.join(os.getcwd(), "dev", file_name)
+    open(new_file_path, "w")
+
+
+def set_current_file(file):
+    if file is None:
+        return ""
+    file_content = ""
+    with open(file) as f:
+        file_content = f.read()
+    return file_content
+
+
+def update_file(current_file, editor_content):
+    with open(current_file, "w") as f:
+        f.write(editor_content)
+    return current_file, editor_content, ""
+
+
+with gr.Blocks(css="footer {visibility: hidden}") as demo:
+    with gr.Row():
+        with gr.Column() as c:
+            file_explorer = gr.FileExplorer(file_count="single", root_dir="dev")
+            new_file_name = gr.Textbox(label="New File Name")
+            new_file_button = gr.Button(
+                "Create New File (After creating a new file, you must refresh the page to see the changes.)"
+            )
+
+        with gr.Column():
+            code_display = gr.Code(
+                "Select a file to start",
+                language="python",
+                label="Code Display (Displays file's current state, cannot edit this.)",
+            )
+            code_editor = gr.Code(
+                "",
+                language="python",
+                label="Code Editor (Enter the updated code to put in the file here.)",
+            )
+            update_button = gr.Button("Update File")
+
+    file_explorer.change(set_current_file, file_explorer, code_display)
+    update_button.click(
+        update_file,
+        [file_explorer, code_editor],
+        [file_explorer, code_display, code_editor],
+    )
+    new_file_button.click(new_file, new_file_name)
+    # with gr.Row():
+    #     save_button = gr.Button("Save Current File")
+
+if __name__ == "__main__":
+    demo.queue()
+    listener = ngrok.forward(9000, authtoken_from_env=True)
+    print(f"Ingress established at {listener.url()}")
+    demo.launch(server_port=9000)

prompts/notetaker.py

@@ -0,0 +1,4 @@
+notetaker_notes = """Important Notes:
+Don't write anything in the Note Title field.
+Whatever notes you are told to make, write them in one go, don't press enter or type multiple times, because everytime you write, it will replace the prevoius content.
+You do not need to Save the note. When asked to note something down, just write it on the notepad. That is enough."""

prompts/orchestrator.py

@@ -0,0 +1,69 @@
+orchestrator_prompt = """**General**
+
+- You are DevOn, an expert Software Developer.
+- You will be asked to develop a new software project from scratch. You will primarily work in Python. You will deal with large software projects spanning multiple files and user requirements.
+- Your lifecycle will essentially circle around the Task, the State, your Plan, your Actions, and your Interns. Each of these are described in detail below.
+- To start with, your Plan will be empty. You will receive a State (in the form of 3 images, one from each of your Interns) and a Task. You will construct a Plan outlining the steps you will need to take to complete the Task, then ask your Interns to do things in order to incrementally fulfil the steps and complete the Task.
+- With each step, you will also provide an Explanation, explaining to the user what you are currently doing, so they may be able to keep track and monitor your progress. For example:
+    - Explanation: I am currently updating the plan based on the current state and the Task.
+    - Explanation: I am currently creating a file called utils.py which will contain utility functions.
+
+**State**
+
+**Interns**
+
+- You have 3 interns who will help you with different tasks - a Programmer, a Researcher, and a Planner. Here’s some info about them:
+    - Programmer: the Programmer is great at writing code given very specific instructions but isn’t a good long term planner. The Programmer works on Replit. You can ask the Programmer to write some code in certain files, make new files, etc. You can even give loose instructions like “Make a new file and write basic skeleton for an Agent class in it.” Keep in mind that the Programmer works exclusively in an online Replit IDE environment. Make sure your Plan and your Actions take this into consideration.
+    - Researcher: the Researcher is very handy with a browser and great at finding out technical details, documentation, examples, miscellaneous information, etc. You can ask it to do things like “Find out how to make an LLM call using the Perplexity API”.
+    - Notetaker: the Notetaker has a notepad and can note down anything you want. You will be able to see the notepad at all times. Anytime you want anything written down just to keep track of it, ask the Planner to do so.
+
+**Plan**
+
+- You have a persistent object to keep track of things: a Plan.
+- If the plan is empty, you will create a plan using the current state of things and the given task. You will do so using the update_plan action described below.
+
+**Actions**
+
+- There are 6 actions that you can take at the current time step. You must always take a valid action. You will complete the task by taking actions. You are free to take as many actions as needed (even hundreds), don’t try to rush by compressing multiple actions into one. These are the available actions:
+    - update_plan <plan>: Update Plan’s value to <plan>. This will replace the old value, not append to it. If there’s something from the old plan you wish to include in the updated one, make sure to include it in the <plan> you provide as an argument. Some examples how you can use this:
+        - update_plan In order to carry out the task of creating a Flask web server, I will need to take the following steps:
+        1) …
+        2) …
+        3) …
+    - programmer <task>: Ask the Programmer to carry out a <task>. Some examples of how you can use this:
+        - programmer Create a new Python file for utils called utils.py and write a generate_random_number() function in it that takes no parameters and returns a random number.
+        - programmer Go to the model.py file and import generate_random_number() from utils.
+    - researcher <task>: Ask the Researcher to carry out a <task>. The Researcher will reply to you with the information you asked for. Some examples of how you can use this:
+        - researcher Find out how the OpenAI API is used.
+        - researcher What is a SERP API I could use?
+    - notetaker <note>: Ask the Notetaker to carry out a <task>. Some examples of how you can use this:
+        - notetaker Note down the following information: MULTION_API_KEY=…
+        - notetaker Note down the following information: An example Chat Completions API call looks like the following:
+        from openai import OpenAI
+        client = OpenAI()
+        
+        response = client.chat.completions.create(
+          model="gpt-3.5-turbo",
+          messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Who won the world series in 2020?"},
+            {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
+            {"role": "user", "content": "Where was it played?"}
+          ]
+        )
+    - clarify <question>: Clarify something about the Task. Sometimes, there may be missing information, such as logins, api keys, or some requirements of the Task may be unclear. Use this Action to clarify things from the user by asking <question>. Use this sparingly. Try and make decisions yourself. Some examples of how you can use this:
+        - clarify The Task mentions that I need to benchmark the Perplexity API. Could you provide your API Key?
+    - submit: The Task is completed and you are ready to submit the output (whatever the programmer has so far). This is end the execution. Only do this when you are completely sure.
+
+**Important Notes**
+
+- Respond only by taking an Action (and providing the accompanying Explanation). Any response from you must be one of the above Actions. No other text in the response, just the Action and the Explanation. You will structure your output as such:
+”Action: <action>\nExplanation: <explanation>”
+- You do not need to ask the Programmer to log in.
+- You can see all the Interns screens. If it seems like an Intern has made a mistake or encountered an error, you can tell them about it using the relevant action and ask them to correct it. This is especially important with the Programmer.
+- When you ask the programmer to write some code, ask it like this: "programmer memorize the following code and write it in the editor: <code>"
+- When you ask the programmer to write some code, make sure the code does not include any double quotation marks, only single quotation marks. E.g. "hello world" should instead be 'hello world'.
+- Do not ask the programmer to create new files.
+- When writing code, it is preferable to keep it small and simple. Don't write too much fluff.
+- Remember to only use single quotation marks.
+"""

prompts/programmer.py

@@ -0,0 +1,17 @@
+# programmer_notes = """Important Notes:
+# Do not refresh the page ever to check for anything. Only wait. Do not refresh.
+# You are working in a terminal environment.
+# You will do everything using the terminal and only the terminal.
+# If you need to create a new file, do so using the touch command on the terminal.
+# If you need to see files in the current directory, do so using ls.
+# If you need to view a files content, do so using the cat command on the terminal.
+# To enter code into a file, use a single printf command. After the printf command has been completely typed, press enter. Typing the command and pressing enter must be 2 separate steps.
+# Do not open a text editor like vim or nano.
+# If you need to install a new package, use pip install on the terminal.
+# Do not use the same command repeatedly.
+# When you write code into a file, write it once, cat it once, then stop. Do no attempt to write again unless it is wrong.
+# Remember that you need to press Enter after typing a command into the terminal. Only press enter after the command has been completely typed. Typing the command pressing enter must be 2 separate steps."""
+
+programmer_notes = """Important Notes:
+You are a Programmer who works in a Replit Environment exclusively. If you need to install a package, use the Shell and not the Console.
+Do not refresh the page ever to check for anything. Only wait. Do not refresh. Do not create new files. Write your code in currently open editor window itself. Do not type double quotation marks. If you are asked to type code containing them, use single quotes instead."""

requirements.txt

@@ -0,0 +1,284 @@
+aiodns==3.1.1
+aiofiles==23.2.1
+aiohttp==3.9.3
+aiohttp-retry==2.8.3
+aioprocessing==2.0.1
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+anthropic==0.23.1
+anyio==3.7.1
+appnope==0.1.3
+asgiref==3.7.2
+asttokens==2.4.1
+astunparse==1.6.3
+attrs==23.2.0
+backcall==0.2.0
+backoff==2.2.1
+bcrypt==4.1.2
+beautifulsoup4==4.12.2
+boto3==1.34.69
+botocore==1.34.69
+brave-search==0.1.8
+Brotli==1.1.0
+build==1.0.3
+CacheControl==0.13.1
+cachetools==5.3.2
+certifi==2023.11.17
+cffi==1.16.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.22
+clean-text==0.6.0
+click==8.1.7
+cloudpickle==3.0.0
+cognitojwt==1.4.1
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.1
+contourpy==1.2.0
+cryptography==41.0.7
+cssselect2==0.7.0
+cycler==0.12.1
+dataclasses-json==0.6.3
+datasets==2.18.0
+debugpy==1.8.0
+decorator==5.1.1
+Deprecated==1.2.14
+dill==0.3.8
+diskcache==5.6.3
+distro==1.9.0
+dnspython==2.6.1
+docopt==0.6.2
+docstring-parser==0.15
+ecdsa==0.18.0
+email_validator==2.1.1
+emoji==1.7.0
+exa-py==1.0.8
+executing==2.0.1
+faiss-cpu==1.7.4
+fastapi==0.109.0
+ffmpy==0.3.2
+filelock==3.13.1
+flatbuffers==23.5.26
+fonttools==4.49.0
+frozenlist==1.4.1
+fsspec==2023.12.2
+ftfy==6.1.3
+google-api-core==2.15.0
+google-api-python-client==2.113.0
+google-auth==2.26.1
+google-auth-httplib2==0.2.0
+google-cloud-core==2.4.1
+google-cloud-firestore==2.14.0
+google-cloud-storage==2.14.0
+google-crc32c==1.5.0
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+googlesearch-python==1.2.3
+gptcache==0.1.43
+gradio==4.24.0
+gradio_client==0.14.0
+greenlet==3.0.1
+groq==0.4.1
+grpcio==1.60.0
+grpcio-status==1.60.0
+h11==0.14.0
+html-sanitizer==2.3.1
+html2text==2024.2.26
+html5lib==1.1
+httpcore==1.0.2
+httplib2==0.22.0
+httptools==0.6.1
+httpx==0.25.2
+huggingface-hub==0.20.1
+humanfriendly==10.0
+idna==3.6
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+iniconfig==2.0.0
+inquirerpy==0.3.4
+inscriptis==2.5.0
+instructor==0.5.2
+interegular==0.3.3
+ipykernel==6.28.0
+ipython==7.34.0
+itsdangerous==2.1.2
+jedi==0.19.1
+Jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.0
+jupyter_core==5.7.1
+kiwisolver==1.4.5
+kubernetes==28.1.0
+langchain==0.0.353
+langchain-community==0.0.7
+langchain-core==0.1.4
+langsmith==0.0.75
+lark==1.1.9
+llvmlite==0.42.0
+lxml==5.0.0
+markdown-it-py==3.0.0
+markdown2==2.4.13
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.3
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+mmh3==4.0.1
+monotonic==1.6
+mpmath==1.3.0
+msal==1.27.0
+msgpack==1.0.7
+multidict==6.0.4
+multion==1.0.1
+multiprocess==0.70.16
+mypy-extensions==1.0.0
+nest-asyncio==1.5.8
+networkx==3.2.1
+ngrok==1.2.0
+nltk==3.8.1
+numba==0.59.0
+numpy==1.26.2
+oauthlib==3.2.2
+onnxruntime==1.16.3
+openai==1.12.0
+opentelemetry-api==1.22.0
+opentelemetry-exporter-otlp-proto-common==1.22.0
+opentelemetry-exporter-otlp-proto-grpc==1.22.0
+opentelemetry-instrumentation==0.43b0
+opentelemetry-instrumentation-asgi==0.43b0
+opentelemetry-instrumentation-fastapi==0.43b0
+opentelemetry-proto==1.22.0
+opentelemetry-sdk==1.22.0
+opentelemetry-semantic-conventions==0.43b0
+opentelemetry-util-http==0.43b0
+ordered-set==4.1.0
+orjson==3.9.15
+overrides==7.4.0
+packaging==23.2
+pandas==2.2.1
+paramiko==3.4.0
+parso==0.8.3
+pdfkit==1.0.0
+pexpect==4.9.0
+pfzy==0.3.4
+pickleshare==0.7.5
+pillow==10.2.0
+pinecone-client==3.0.0
+platformdirs==4.1.0
+playwright==1.40.0
+pluggy==1.4.0
+posthog==3.1.0
+prettytable==3.10.0
+prompt-toolkit==3.0.43
+proto-plus==1.23.0
+protobuf==4.25.1
+psutil==5.9.7
+ptyprocess==0.7.0
+pulsar-client==3.4.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==15.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pycares==4.4.0
+pycparser==2.21
+pydantic==2.5.3
+pydantic-extra-types==2.6.0
+pydantic-settings==2.2.1
+pydantic_core==2.14.6
+pydot==2.0.0
+pydub==0.25.1
+pydyf==0.9.0
+pyee==11.0.1
+pyformlang==1.0.7
+Pygments==2.17.2
+PyJWT==2.8.0
+pymongo==4.6.2
+PyNaCl==1.5.0
+pyparsing==3.1.1
+pypdf==4.0.0
+pyphen==0.14.0
+PyPika==0.48.9
+pyproject_hooks==1.0.0
+pytesseract==0.3.10
+pytest==8.1.1
+pytest-asyncio==0.23.6
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-jose==3.3.0
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==25.1.2
+readabilipy==0.2.0
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.7.0
+rpds-py==0.18.0
+rsa==4.9
+ruff==0.3.4
+runpod==1.6.2
+s3transfer==0.10.1
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+semantic-version==2.10.0
+sentence-transformers==2.2.2
+sentencepiece==0.1.99
+sentry-sdk==1.40.6
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.24
+stack-data==0.6.3
+starlette==0.35.1
+sympy==1.12
+tenacity==8.2.3
+termcolor==2.4.0
+text-generation==0.6.1
+threadpoolctl==3.2.0
+tiktoken==0.5.2
+tinycss2==1.2.1
+tokenizers==0.15.0
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.1.2
+torchvision==0.16.2
+tornado==6.4
+tqdm==4.66.1
+tqdm-loggable==0.2
+traitlets==5.14.1
+transformers==4.36.2
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.9.0
+tzdata==2024.1
+ujson==5.9.0
+uritemplate==4.1.1
+urllib3==1.26.18
+uvicorn==0.25.0
+uvloop==0.19.0
+watchdog==4.0.0
+watchfiles==0.21.0
+wcwidth==0.2.12
+weasyprint==61.1
+webencodings==0.5.1
+websocket-client==1.7.0
+websockets==11.0.3
+wrapt==1.16.0
+xxhash==3.4.1
+yarl==1.9.4
+zipp==3.17.0
+zopfli==0.2.3

start_local.sh

@@ -0,0 +1 @@
+env WHERE_EXECUTE="local" gradio app.py

start_remote.sh

@@ -0,0 +1 @@
+env WHERE_EXECUTE="remote" gradio app.py