add code interpreter from open-interpreter

code_interpreters/base_code_interpreter.py

@@ -0,0 +1,14 @@
+
+
+class BaseCodeInterpreter:
+    """
+    .run is a generator that yields a dict with attributes: active_line, output
+    """
+    def __init__(self):
+        pass
+
+    def run(self, code):
+        pass
+
+    def terminate(self):
+        pass

code_interpreters/create_code_interpreter.py

@@ -0,0 +1,11 @@
+from .language_map import language_map
+
+def create_code_interpreter(language):
+    # Case in-sensitive
+    language = language.lower()
+
+    try:
+        CodeInterpreter = language_map[language]
+        return CodeInterpreter()
+    except KeyError:
+        raise ValueError(f"Unknown or unsupported language: {language}")

code_interpreters/language_map.py

@@ -0,0 +1,19 @@
+from .languages.python import Python
+from .languages.shell import Shell
+from .languages.javascript import JavaScript
+from .languages.html import HTML
+from .languages.applescript import AppleScript
+from .languages.r import R
+from .languages.powershell import PowerShell
+
+
+language_map = {
+    "python": Python,
+    "bash": Shell,
+    "shell": Shell,
+    "javascript": JavaScript,
+    "html": HTML,
+    "applescript": AppleScript,
+    "r": R,
+    "powershell": PowerShell,
+}

code_interpreters/languages/applescript.py

@@ -0,0 +1,64 @@
+import os
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+class AppleScript(SubprocessCodeInterpreter):
+    file_extension = "applescript"
+    proper_name = "AppleScript"
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = os.environ.get('SHELL', '/bin/zsh')
+
+    def preprocess_code(self, code):
+        """
+        Inserts an end_of_execution marker and adds active line indicators.
+        """
+        # Add active line indicators to the code
+        code = self.add_active_line_indicators(code)
+
+        # Escape double quotes
+        code = code.replace('"', r'\"')
+        
+        # Wrap in double quotes
+        code = '"' + code + '"'
+        
+        # Prepend start command for AppleScript
+        code = "osascript -e " + code
+
+        # Append end of execution indicator
+        code += '; echo "## end_of_execution ##"'
+        
+        return code
+
+    def add_active_line_indicators(self, code):
+        """
+        Adds log commands to indicate the active line of execution in the AppleScript.
+        """
+        modified_lines = []
+        lines = code.split('\n')
+
+        for idx, line in enumerate(lines):
+            # Add log command to indicate the line number
+            if line.strip():  # Only add if line is not empty
+                modified_lines.append(f'log "## active_line {idx + 1} ##"')
+            modified_lines.append(line)
+
+        return '\n'.join(modified_lines)
+
+    def detect_active_line(self, line):
+        """
+        Detects active line indicator in the output.
+        """
+        prefix = "## active_line "
+        if prefix in line:
+            try:
+                return int(line.split(prefix)[1].split()[0])
+            except:
+                pass
+        return None
+
+    def detect_end_of_execution(self, line):
+        """
+        Detects end of execution marker in the output.
+        """
+        return "## end_of_execution ##" in line

code_interpreters/languages/html.py

@@ -0,0 +1,21 @@
+import webbrowser
+import tempfile
+import os
+from ..base_code_interpreter import BaseCodeInterpreter
+
+class HTML(BaseCodeInterpreter):
+    file_extension = "html"
+    proper_name = "HTML"
+
+    def __init__(self):
+        super().__init__()
+
+    def run(self, code):
+        # Create a temporary HTML file with the content
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".html") as f:
+            f.write(code.encode())
+
+        # Open the HTML file with the default web browser
+        webbrowser.open('file://' + os.path.realpath(f.name))
+
+        yield {"output": f"Saved to {os.path.realpath(f.name)} and opened with the user's default web browser."}

code_interpreters/languages/javascript.py

@@ -0,0 +1,64 @@
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+import re
+
+class JavaScript(SubprocessCodeInterpreter):
+    file_extension = "js"
+    proper_name = "JavaScript"
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = "node -i"
+        
+    def preprocess_code(self, code):
+        return preprocess_javascript(code)
+    
+    def line_postprocessor(self, line):
+        # Node's interactive REPL outputs a billion things
+        # So we clean it up:
+        if "Welcome to Node.js" in line:
+            return None
+        if line.strip() in ["undefined", 'Type ".help" for more information.']:
+            return None
+        # Remove trailing ">"s
+        line = re.sub(r'^\s*(>\s*)+', '', line)
+        return line
+
+    def detect_active_line(self, line):
+        if "## active_line " in line:
+            return int(line.split("## active_line ")[1].split(" ##")[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return "## end_of_execution ##" in line
+    
+
+def preprocess_javascript(code):
+    """
+    Add active line markers
+    Wrap in a try catch
+    Add end of execution marker
+    """
+
+    # Split code into lines
+    lines = code.split("\n")
+    processed_lines = []
+
+    for i, line in enumerate(lines, 1):
+        # Add active line print
+        processed_lines.append(f'console.log("## active_line {i} ##");')
+        processed_lines.append(line)
+
+    # Join lines to form the processed code
+    processed_code = "\n".join(processed_lines)
+
+    # Wrap in a try-catch and add end of execution marker
+    processed_code = f"""
+try {{
+{processed_code}
+}} catch (e) {{
+    console.log(e);
+}}
+console.log("## end_of_execution ##");
+"""
+
+    return processed_code

code_interpreters/languages/powershell.py

@@ -0,0 +1,68 @@
+import platform
+import os
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+class PowerShell(SubprocessCodeInterpreter):
+    file_extension = "ps1"
+    proper_name = "PowerShell"
+
+    def __init__(self):
+        super().__init__()
+
+        # Determine the start command based on the platform (use "powershell" for Windows)
+        if platform.system() == 'Windows':
+            self.start_cmd = 'powershell.exe'
+            #self.start_cmd = os.environ.get('SHELL', 'powershell.exe')
+        else:
+            self.start_cmd = os.environ.get('SHELL', 'bash')
+
+    def preprocess_code(self, code):
+        return preprocess_powershell(code)
+
+    def line_postprocessor(self, line):
+        return line
+
+    def detect_active_line(self, line):
+        if "## active_line " in line:
+            return int(line.split("## active_line ")[1].split(" ##")[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return "## end_of_execution ##" in line
+
+def preprocess_powershell(code):
+    """
+    Add active line markers
+    Wrap in try-catch block
+    Add end of execution marker
+    """
+    # Add commands that tell us what the active line is
+    code = add_active_line_prints(code)
+
+    # Wrap in try-catch block for error handling
+    code = wrap_in_try_catch(code)
+
+    # Add end marker (we'll be listening for this to know when it ends)
+    code += '\nWrite-Output "## end_of_execution ##"'
+
+    return code
+
+def add_active_line_prints(code):
+    """
+    Add Write-Output statements indicating line numbers to a PowerShell script.
+    """
+    lines = code.split('\n')
+    for index, line in enumerate(lines):
+        # Insert the Write-Output command before the actual line
+        lines[index] = f'Write-Output "## active_line {index + 1} ##"\n{line}'
+    return '\n'.join(lines)
+
+def wrap_in_try_catch(code):
+    """
+    Wrap PowerShell code in a try-catch block to catch errors and display them.
+    """
+    try_catch_code = """
+try {
+    $ErrorActionPreference = "Stop"
+"""
+    return try_catch_code + code + "\n} catch {\n    Write-Error $_\n}\n"

code_interpreters/languages/python.py

@@ -0,0 +1,157 @@
+import os
+import sys
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+import ast
+import re
+import shlex
+
+class Python(SubprocessCodeInterpreter):
+    file_extension = "py"
+    proper_name = "Python"
+
+    def __init__(self):
+        super().__init__()
+        executable = sys.executable
+        if os.name != 'nt':  # not Windows
+            executable = shlex.quote(executable)
+        self.start_cmd = executable + " -i -q -u"
+        
+    def preprocess_code(self, code):
+        return preprocess_python(code)
+    
+    def line_postprocessor(self, line):
+        if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*)', line):
+            return None
+        return line
+
+    def detect_active_line(self, line):
+        if "## active_line " in line:
+            return int(line.split("## active_line ")[1].split(" ##")[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return "## end_of_execution ##" in line
+    
+
+def preprocess_python(code):
+    """
+    Add active line markers
+    Wrap in a try except
+    Add end of execution marker
+    """
+
+    # Add print commands that tell us what the active line is
+    code = add_active_line_prints(code)
+
+    # Wrap in a try except
+    code = wrap_in_try_except(code)
+
+    # Remove any whitespace lines, as this will break indented blocks
+    # (are we sure about this? test this)
+    code_lines = code.split("\n")
+    code_lines = [c for c in code_lines if c.strip() != ""]
+    code = "\n".join(code_lines)
+
+    # Add end command (we'll be listening for this so we know when it ends)
+    code += '\n\nprint("## end_of_execution ##")'
+
+    return code
+
+
+def add_active_line_prints(code):
+    """
+    Add print statements indicating line numbers to a python string.
+    """
+    tree = ast.parse(code)
+    transformer = AddLinePrints()
+    new_tree = transformer.visit(tree)
+    return ast.unparse(new_tree)
+
+
+class AddLinePrints(ast.NodeTransformer):
+    """
+    Transformer to insert print statements indicating the line number
+    before every executable line in the AST.
+    """
+
+    def insert_print_statement(self, line_number):
+        """Inserts a print statement for a given line number."""
+        return ast.Expr(
+            value=ast.Call(
+                func=ast.Name(id='print', ctx=ast.Load()),
+                args=[ast.Constant(value=f"## active_line {line_number} ##")],
+                keywords=[]
+            )
+        )
+
+    def process_body(self, body):
+        """Processes a block of statements, adding print calls."""
+        new_body = []
+
+        # In case it's not iterable:
+        if not isinstance(body, list):
+            body = [body]
+
+        for sub_node in body:
+            if hasattr(sub_node, 'lineno'):
+                new_body.append(self.insert_print_statement(sub_node.lineno))
+            new_body.append(sub_node)
+
+        return new_body
+
+    def visit(self, node):
+        """Overridden visit to transform nodes."""
+        new_node = super().visit(node)
+
+        # If node has a body, process it
+        if hasattr(new_node, 'body'):
+            new_node.body = self.process_body(new_node.body)
+
+        # If node has an orelse block (like in for, while, if), process it
+        if hasattr(new_node, 'orelse') and new_node.orelse:
+            new_node.orelse = self.process_body(new_node.orelse)
+
+        # Special case for Try nodes as they have multiple blocks
+        if isinstance(new_node, ast.Try):
+            for handler in new_node.handlers:
+                handler.body = self.process_body(handler.body)
+            if new_node.finalbody:
+                new_node.finalbody = self.process_body(new_node.finalbody)
+
+        return new_node
+    
+
+def wrap_in_try_except(code):
+    # Add import traceback
+    code = "import traceback\n" + code
+
+    # Parse the input code into an AST
+    parsed_code = ast.parse(code)
+
+    # Wrap the entire code's AST in a single try-except block
+    try_except = ast.Try(
+        body=parsed_code.body,
+        handlers=[
+            ast.ExceptHandler(
+                type=ast.Name(id="Exception", ctx=ast.Load()),
+                name=None,
+                body=[
+                    ast.Expr(
+                        value=ast.Call(
+                            func=ast.Attribute(value=ast.Name(id="traceback", ctx=ast.Load()), attr="print_exc", ctx=ast.Load()),
+                            args=[],
+                            keywords=[]
+                        )
+                    ),
+                ]
+            )
+        ],
+        orelse=[],
+        finalbody=[]
+    )
+
+    # Assign the try-except block as the new body
+    parsed_code.body = [try_except]
+
+    # Convert the modified AST back to source code
+    return ast.unparse(parsed_code)

code_interpreters/languages/r.py

@@ -0,0 +1,67 @@
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+import re
+
+class R(SubprocessCodeInterpreter):
+    file_extension = "r"
+    proper_name = "R"
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = "R -q --vanilla"  # Start R in quiet and vanilla mode
+        
+    def preprocess_code(self, code):
+        """
+        Add active line markers
+        Wrap in a tryCatch for better error handling in R
+        Add end of execution marker
+        """
+
+        lines = code.split("\n")
+        processed_lines = []
+
+        for i, line in enumerate(lines, 1):
+            # Add active line print
+            processed_lines.append(f'cat("## active_line {i} ##\\n");{line}')
+
+        # Join lines to form the processed code
+        processed_code = "\n".join(processed_lines)
+
+        # Wrap in a tryCatch for error handling and add end of execution marker
+        processed_code = f"""
+tryCatch({{
+{processed_code}
+}}, error=function(e){{
+    cat("## execution_error ##\\n", conditionMessage(e), "\\n");
+}})
+cat("## end_of_execution ##\\n");
+"""
+        # Count the number of lines of processed_code
+        # (R echoes all code back for some reason, but we can skip it if we track this!)
+        self.code_line_count = len(processed_code.split("\n")) - 1
+        
+        return processed_code
+    
+    def line_postprocessor(self, line):
+        # If the line count attribute is set and non-zero, decrement and skip the line
+        if hasattr(self, "code_line_count") and self.code_line_count > 0:
+            self.code_line_count -= 1
+            return None
+
+        if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*|\s*>\s*|\s*\+\s*|\s*)$', line):
+            return None
+        if "R version" in line:  # Startup message
+            return None
+        if line.strip().startswith("[1] \"") and line.endswith("\""):  # For strings, trim quotation marks
+            return line[5:-1].strip()
+        if line.strip().startswith("[1]"):  # Normal R output prefix for non-string outputs
+            return line[4:].strip()
+
+        return line
+
+    def detect_active_line(self, line):
+        if "## active_line " in line:
+            return int(line.split("## active_line ")[1].split(" ##")[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return "## end_of_execution ##" in line or "## execution_error ##" in line

code_interpreters/languages/shell.py

@@ -0,0 +1,57 @@
+import platform
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+import os
+
+class Shell(SubprocessCodeInterpreter):
+    file_extension = "sh"
+    proper_name = "Shell"
+
+    def __init__(self):
+        super().__init__()
+
+        # Determine the start command based on the platform
+        if platform.system() == 'Windows':
+            self.start_cmd = 'cmd.exe'
+        else:
+            self.start_cmd = os.environ.get('SHELL', 'bash')
+
+    def preprocess_code(self, code):
+        return preprocess_shell(code)
+    
+    def line_postprocessor(self, line):
+        return line
+
+    def detect_active_line(self, line):
+        if "## active_line " in line:
+            return int(line.split("## active_line ")[1].split(" ##")[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return "## end_of_execution ##" in line
+        
+
+def preprocess_shell(code):
+    """
+    Add active line markers
+    Wrap in a try except (trap in shell)
+    Add end of execution marker
+    """
+    
+    # Add commands that tell us what the active line is
+    code = add_active_line_prints(code)
+    
+    # Add end command (we'll be listening for this so we know when it ends)
+    code += '\necho "## end_of_execution ##"'
+    
+    return code
+
+
+def add_active_line_prints(code):
+    """
+    Add echo statements indicating line numbers to a shell string.
+    """
+    lines = code.split('\n')
+    for index, line in enumerate(lines):
+        # Insert the echo command before the actual line
+        lines[index] = f'echo "## active_line {index + 1} ##"\n{line}'
+    return '\n'.join(lines)

code_interpreters/subprocess_code_interpreter.py

@@ -0,0 +1,138 @@
+
+
+import subprocess
+import threading
+import queue
+import time
+import traceback
+from .base_code_interpreter import BaseCodeInterpreter
+
+class SubprocessCodeInterpreter(BaseCodeInterpreter):
+    def __init__(self):
+        self.start_cmd = ""
+        self.process = None
+        self.debug_mode = False
+        self.output_queue = queue.Queue()
+        self.done = threading.Event()
+
+    def detect_active_line(self, line):
+        return None
+    
+    def detect_end_of_execution(self, line):
+        return None
+    
+    def line_postprocessor(self, line):
+        return line
+    
+    def preprocess_code(self, code):
+        """
+        This needs to insert an end_of_execution marker of some kind,
+        which can be detected by detect_end_of_execution.
+
+        Optionally, add active line markers for detect_active_line.
+        """
+        return code
+    
+    def terminate(self):
+        self.process.terminate()
+
+    def start_process(self):
+        if self.process:
+            self.terminate()
+
+        self.process = subprocess.Popen(self.start_cmd.split(),
+                                        stdin=subprocess.PIPE,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE,
+                                        text=True,
+                                        bufsize=0,
+                                        universal_newlines=True)
+        threading.Thread(target=self.handle_stream_output,
+                            args=(self.process.stdout, False),
+                            daemon=True).start()
+        threading.Thread(target=self.handle_stream_output,
+                            args=(self.process.stderr, True),
+                            daemon=True).start()
+
+    def run(self, code):
+        retry_count = 0
+        max_retries = 3
+
+        # Setup
+        try:
+            code = self.preprocess_code(code)
+            if not self.process:
+                self.start_process()
+        except:
+            yield {"output": traceback.format_exc()}
+            return
+            
+
+        while retry_count <= max_retries:
+            if self.debug_mode:
+                print(f"Running code:\n{code}\n---")
+
+            self.done.clear()
+
+            try:
+                self.process.stdin.write(code + "\n")
+                self.process.stdin.flush()
+                break
+            except:
+                if retry_count != 0:
+                    # For UX, I like to hide this if it happens once. Obviously feels better to not see errors
+                    # Most of the time it doesn't matter, but we should figure out why it happens frequently with:
+                    # applescript
+                    yield {"output": traceback.format_exc()}
+                    yield {"output": f"Retrying... ({retry_count}/{max_retries})"}
+                    yield {"output": "Restarting process."}
+
+                self.start_process()
+
+                retry_count += 1
+                if retry_count > max_retries:
+                    yield {"output": "Maximum retries reached. Could not execute code."}
+                    return
+
+        while True:
+            if not self.output_queue.empty():
+                yield self.output_queue.get()
+            else:
+                time.sleep(0.1)
+            try:
+                output = self.output_queue.get(timeout=0.3)  # Waits for 0.3 seconds
+                yield output
+            except queue.Empty:
+                if self.done.is_set():
+                    # Try to yank 3 more times from it... maybe there's something in there...
+                    # (I don't know if this actually helps. Maybe we just need to yank 1 more time)
+                    for _ in range(3):
+                        if not self.output_queue.empty():
+                            yield self.output_queue.get()
+                        time.sleep(0.2)
+                    break
+
+    def handle_stream_output(self, stream, is_error_stream):
+        for line in iter(stream.readline, ''):
+            if self.debug_mode:
+                print(f"Received output line:\n{line}\n---")
+
+            line = self.line_postprocessor(line)
+
+            if line is None:
+                continue # `line = None` is the postprocessor's signal to discard completely
+
+            if self.detect_active_line(line):
+                active_line = self.detect_active_line(line)
+                self.output_queue.put({"active_line": active_line})
+            elif self.detect_end_of_execution(line):
+                self.output_queue.put({"active_line": None})
+                time.sleep(0.1)
+                self.done.set()
+            elif is_error_stream and "KeyboardInterrupt" in line:
+                self.output_queue.put({"output": "KeyboardInterrupt"})
+                time.sleep(0.1)
+                self.done.set()
+            else:
+                self.output_queue.put({"output": line})
+