initial commit

.gitignore

@@ -0,0 +1 @@
+**/__pycache__/

README.md

@@ -1,13 +1,12 @@
 ---
 title: CodeLATS
-emoji: 📚
-colorFrom: pink
-colorTo: gray
+emoji: 🐢
+colorFrom: gray
+colorTo: yellow
 sdk: streamlit
-sdk_version: 1.27.2
+sdk_version: 1.27.1
 app_file: app.py
 pinned: false
-license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,4 +1,109 @@
 import streamlit as st
+import openai
+import os
+import sys
+import argparse
+sys.path.append('./LATS')
+from lats_main import lats_main
+
+st.set_page_config(layout="wide")
+
+# Initialize session state variables if they don't exist.
+if 'response_content' not in st.session_state:
+    st.session_state.response_content = None
+
+# Creating main columns for the chat and runtime notifications
+chat_col = st.container()
+
+chat_col.title("CodeLATS")
+description = """This tech demo is an implementation of Language Agent Tree Search (LATS) (https://arxiv.org/abs/2310.04406) built specifically for generating code in the form of python functions. It achieves **state-of-the-art** results on HumanEval with a **94.4% pass@1 rate** on GPT-4.
+
+Listed below is an example programming problem (https://leetcode.com/problems/longest-valid-parentheses/description/) to get started with. 
+
+```python
+Given a string containing just the characters '(' and ')', return the length of the longest valid (well-formed) parentheses substring
+```
+NOTE: On average a call for a HumanEval or Leetcode question will cost around 5-30 cents on GPT-4, using the default parameters. This value may change depending on problem difficulty and parameters.
+"""
+
+chat_col.markdown(description)
+sidebar = st.sidebar
+# Runtime Section
+runtime_container = st.container()
+
+# Parameters Section
+sidebar.title("**An AI@UIUC Project** (https://uiuc.ai/)")
+parameters_section = sidebar.expander("Parameters", expanded=False)
+tree_width = parameters_section.number_input("Tree Width", min_value=1, max_value=5, value=1)
+tree_depth = parameters_section.number_input("Tree Depth", min_value=1, max_value=8, value=3)
+iterations = parameters_section.number_input("Iterations", min_value=1, max_value=4, value=2)
+key = st.sidebar.text_input("Enter your OpenAI Api Key:", type="password")
+sidebar.markdown('<hr style="margin-top: 0.5rem; margin-bottom: 0.5rem;">', unsafe_allow_html=True)
+
+with sidebar:
+    runtime_container = st.container()
+    runtime_container.empty()
+
+runtime_messages = []
+
+def make_args(instruction, tree_depth, tree_width, iterations):
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--strategy", default="mcts", help="Strategy to use")
+    parser.add_argument("--language", default="py", help="Programming language")
+    parser.add_argument("--model", default="gpt-4", help="Model type")
+    parser.add_argument("--max_iters", default=iterations, help="Maximum iterations")
+    parser.add_argument("--instruction", default=instruction, help="Instruction text")
+    parser.add_argument("--verbose", action="store_true", help="Verbose output")
+    parser.add_argument("--is_leetcode", action='store_true',
+                        help="To run the leetcode benchmark")  # Temporary
+    parser.add_argument("--n_samples", type=int,
+                        help="The number of nodes added during expansion", default=tree_width)
+    parser.add_argument("--depth", type=int,
+                        help="Tree depth", default=tree_depth)
+    args = parser.parse_args()
+    return args
+
+def run_querry():
+    if user_input:
+        
+        # Create a new container for each subsequent message
+        runtime_container.write("Initiating process...")
+
+        # Make it so that prints go to runtime_container writes instead
+        old_stdout = sys.stdout
+        sys.stdout = runtime_container
+
+        with chat_col:
+
+            with st.spinner('Running...'):
+                args = make_args(user_input, tree_depth, tree_width, iterations)
+                # main call
+                response = lats_main(args)
+
+        sys.stdout = old_stdout
+        runtime_container.write("Response fetched.")
+        chat_col.markdown('<hr style="margin-top: 0.5rem; margin-bottom: 0.5rem;">', unsafe_allow_html=True)
+        chat_col.write(f"```python\n{response} \n")
+
+        return response
+
+# User input section at the bottom of the page
+with chat_col:
+    user_input = st.text_area("Enter your message here:", placeholder="Type your message here...", label_visibility="collapsed")
+    button = st.button("Send")
+
+    if button:
+        fail = False
+        if key == "":
+            st.warning("Missing OpenAI API Key")
+            fail = True
+
+        if user_input == "":
+            st.warning("Missing a coding problem")
+            fail = True
+        
+        if (not fail):
+            openai.api_key = key
+            run_querry()
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)

executors/__init__.py

@@ -0,0 +1,2 @@
+from .py_executor import PyExecutor
+from .factory import executor_factory

executors/cargo_harness/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "cargo_harness"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

executors/executor_types.py

@@ -0,0 +1,20 @@
+from typing import NamedTuple, List, Tuple
+from abc import ABC, abstractmethod
+
+class ExecuteResult(NamedTuple):
+    is_passing: bool
+    feedback: str
+    state: Tuple[bool]
+
+class Executor(ABC):
+    @abstractmethod
+    def execute(self, func: str, tests: List[str], timeout: int = 5) -> ExecuteResult:
+        ...
+
+    @abstractmethod
+    def evaluate(self, name: str, func: str, test: str, timeout: int = 5) -> bool:
+        ...
+
+
+
+

executors/executor_utils.py

@@ -0,0 +1,46 @@
+
+def timeout_handler(_, __):
+    raise TimeoutError()
+
+import os, json
+def to_jsonl(dict_data, file_path):
+    with open(file_path, 'a') as file:
+        json_line = json.dumps(dict_data)
+        file.write(json_line + os.linesep)
+
+from threading import Thread
+class PropagatingThread(Thread):
+    def run(self):
+        self.exc = None
+        try:
+            if hasattr(self, '_Thread__target'):
+                # Thread uses name mangling prior to Python 3.
+                self.ret = self._Thread__target(*self._Thread__args, **self._Thread__kwargs)
+            else:
+                self.ret = self._target(*self._args, **self._kwargs)
+        except BaseException as e:
+            self.exc = e
+
+    def join(self, timeout=None):
+        super(PropagatingThread, self).join(timeout)
+        if self.exc:
+            raise self.exc
+        return self.ret
+    
+
+def function_with_timeout(func, args, timeout):
+    result_container = []
+
+    def wrapper():
+        result_container.append(func(*args))
+
+    thread = PropagatingThread(target=wrapper)
+    thread.start()
+    thread.join(timeout)
+
+    if thread.is_alive():
+        raise TimeoutError()
+    else:
+        return result_container[0]
+
+

executors/factory.py

@@ -0,0 +1,8 @@
+from .py_executor import PyExecutor
+from .executor_types import Executor
+
+def executor_factory(lang: str) -> Executor:
+    if lang == "py" or lang == "python":
+        return PyExecutor()
+    else:
+        raise ValueError(f"Invalid language for executor: {lang}")

executors/py_executor.py

@@ -0,0 +1,88 @@
+import ast
+import signal
+import astunparse
+
+from .executor_utils import function_with_timeout
+
+from typing import List
+from .executor_types import ExecuteResult, Executor
+
+class PyExecutor(Executor):
+    def execute(self, func: str, tests: List[str], timeout: int = 5) -> ExecuteResult:
+        # Combine function code and assert statement
+        imports = 'from typing import *'
+        func_test_list = [f'{imports}\n{func}\n{test}' for test in tests]
+
+        # Run the tests and collect the results
+        success_tests = []
+        failed_tests = []
+        is_passing = True
+        num_tests = len(func_test_list)
+        for i in range(num_tests):
+            try:
+
+                function_with_timeout(exec, (func_test_list[i], globals()), timeout)
+
+                success_tests += [tests[i]]
+            except Exception:
+                output = get_output(func, tests[i], timeout=timeout)
+                failed_tests += [f"{tests[i]} # output: {output}"]
+                is_passing = False
+
+        state = []
+        for test in tests:
+            if test in success_tests:
+                state += [True]
+            else:
+                state += [False]
+
+        state = tuple(state)
+
+        feedback = "Tested passed:"
+        for test in success_tests:
+            feedback += f"\n{test}"
+        feedback += "\n\nTests failed:"
+        for test in failed_tests:
+            feedback += f"\n{test}"
+            
+        return ExecuteResult(is_passing, feedback, state)
+
+    def evaluate(self, name: str, func: str, test: str, timeout: int = 5) -> bool:
+        """
+        Evaluates the implementation on Human-Eval Python.
+
+        probably should be written in a dataset-agnostic way but not now
+        """
+        code = f"""{func}
+
+{test}
+
+check({name})
+    """
+        try:
+
+            function_with_timeout(exec, (code, globals()), timeout)
+
+            return True
+        except Exception:
+            return False
+
+def get_call_str(assert_statement: str) -> str:
+    ast_parsed = ast.parse(assert_statement)
+    try:
+        call_str = ast_parsed.body[0].test.left # type: ignore
+    except:
+        call_str = ast_parsed.body[0].test # type: ignore
+
+    return astunparse.unparse(call_str).strip()
+
+def get_output(func: str, assert_statement: str, timeout: int = 5) -> str:
+    try:
+        exec(f"from typing import *\n{func}", globals())
+        func_call = get_call_str(assert_statement)
+        output = function_with_timeout(eval, (func_call, globals()), timeout)
+        return output
+    except TimeoutError:
+        return "TIMEOUT"
+    except Exception as e:
+        return str(e)

generators/__init__.py

@@ -0,0 +1,3 @@
+from .py_generate import PyGenerator
+from .factory import generator_factory, model_factory
+from .model import ModelBase, GPT4, GPT35

generators/factory.py

@@ -0,0 +1,20 @@
+from .py_generate import PyGenerator
+from .generator_types import Generator
+from .model import ModelBase, GPT4, GPT35, GPTDavinci
+
+def generator_factory(lang: str) -> Generator:
+    if lang == "py" or lang == "python":
+        return PyGenerator()
+    else:
+        raise ValueError(f"Invalid language for generator: {lang}")
+
+
+def model_factory(model_name: str) -> ModelBase:
+    if model_name == "gpt-4":
+        return GPT4()
+    elif model_name == "gpt-3.5-turbo-0613":
+        return GPT35()
+    elif model_name.startswith("text-davinci"):
+        return GPTDavinci(model_name)
+    else:
+        raise ValueError(f"Invalid model name: {model_name}")

generators/generator_types.py

@@ -0,0 +1,33 @@
+from typing import List, Optional, Union
+from abc import abstractmethod, ABC
+
+from generators.model import ModelBase
+
+
+class Generator:
+    @abstractmethod
+    def self_reflection(self, func: str, feedback: str, model: ModelBase) -> str:
+        ...
+
+    @abstractmethod
+    def func_impl(
+        self,
+        func_sig: str,
+        model: ModelBase,
+        strategy: str,
+        prev_func_impl: Optional[str] = None,
+        feedback: Optional[str] = None,
+        self_reflection: Optional[str] = None,
+        num_comps: int = 1,
+        temperature: float = 0.0,
+    ) -> Union[str, List[str]]:
+        ...
+
+    @abstractmethod
+    def internal_tests(
+            self,
+            func_sig: str,
+            model: ModelBase,
+            max_num_tests: int = 5
+    ) -> List[str]:
+        ...

generators/generator_utils.py

@@ -0,0 +1,286 @@
+from generators.model import ModelBase, Message
+import random
+import streamlit as st
+
+from typing import Union, List, Optional, Callable
+
+
+def generic_generate_func_impl(
+    func_sig: str,
+    model: ModelBase,
+    strategy: str,
+    prev_func_impl,
+    feedback,
+    self_reflection,
+    num_comps,
+    temperature,
+    reflexion_chat_instruction: str,
+    reflexion_few_shot: str,
+    simple_chat_instruction: str,
+    reflexion_completion_instruction: str,
+    simple_completion_instruction: str,
+    code_block_instruction: str,
+    parse_code_block: Callable[[str], str],
+    add_code_block: Callable[[str], str]
+) -> Union[str, List[str]]:
+    if strategy != "reflexion" and strategy != "simple":
+        raise ValueError(
+            f"Invalid strategy: given `{strategy}` but expected one of `reflexion` or `simple`")
+    if strategy == "reflexion" and (prev_func_impl is None or feedback is None or self_reflection is None):
+        raise ValueError(
+            f"Invalid arguments: given `strategy=reflexion` but `prev_func_impl`, `feedback`, or `self_reflection` is None")
+
+    if model.is_chat:
+        if strategy == "reflexion":
+            message = f"{reflexion_few_shot}\n[previous impl]:\n{add_code_block(prev_func_impl)}\n\n[unit test results from previous impl]:\n{feedback}\n\n[reflection on previous impl]:\n{self_reflection}\n\n[improved impl]:\n{func_sig}"
+            prompt = f"{reflexion_chat_instruction}\n{code_block_instruction}"
+            # func_bodies is a really bad name, as it can also be just 1 string
+            print_messages(prompt, message)
+            messages = [
+                Message(
+                    role="system",
+                    content=prompt,
+                ),
+                Message(
+                    role="user", # TODO: check this
+                    content=reflexion_few_shot,
+                ),
+                Message(
+                    role="assistant",
+                    content=add_code_block(prev_func_impl),
+                ),
+                Message(
+                    role="user",
+                    content=f"[unit test results from previous impl]:\n{feedback}\n\n[reflection on previous impl]:",
+                ),
+                Message(
+                    role="assistant",
+                    content=self_reflection,
+                ),
+                Message(
+                    role="user",
+                    content=f"[improved impl]:\n{func_sig}",
+                ),
+            ]
+            func_bodies = model.generate_chat(messages=messages, num_comps=num_comps, temperature=temperature)
+        else:
+            system_prompt = f"{simple_chat_instruction}\n{code_block_instruction}"
+            print_messages(system_prompt, func_sig)
+            messages = [
+                Message(
+                    role="system",
+                    content=f"{simple_chat_instruction}\n{code_block_instruction}",
+                ),
+                Message(
+                    role="user",
+                    content=func_sig,
+                ),
+            ]
+            func_bodies = model.generate_chat(messages=messages, num_comps=num_comps, temperature=temperature)
+    else:
+        if strategy == "reflexion":
+            prompt = f"{reflexion_completion_instruction}\n{add_code_block(prev_func_impl)}\n\nunit tests:\n{feedback}\n\nhint:\n{self_reflection}\n\n# improved implementation\n{func_sig}\n{code_block_instruction}"
+            func_bodies = model.generate(
+                prompt, num_comps=num_comps, temperature=temperature)
+        else:
+            prompt = f"{simple_completion_instruction}\n{func_sig}\n{code_block_instruction}"
+            func_bodies = model.generate(
+                prompt, num_comps=num_comps, temperature=temperature)
+
+    if num_comps == 1:
+        assert isinstance(func_bodies, str)
+        func_body_str = parse_code_block(func_bodies)
+        print_generated_func_body(func_body_str)
+        return func_body_str
+
+    else:
+        func_bodies = [parse_code_block(func_body) for func_body in func_bodies]
+        print_generated_func_body("\n\n".join(func_bodies))
+        return func_bodies
+
+
+def generate_with_accumulated_context(
+    func_sig: str,
+    model: ModelBase,
+    strategy: str,
+    prev_func_impl,
+    accumulated_feedback,
+    accumulated_reflection,
+    num_comps,
+    temperature,
+    reflexion_chat_instruction: str,
+    reflexion_few_shot: str,
+    simple_chat_instruction: str,
+    reflexion_completion_instruction: str,
+    simple_completion_instruction: str,
+    code_block_instruction: str,
+    parse_code_block: Callable[[str], str],
+    add_code_block: Callable[[str], str]
+) -> Union[str, List[str]]:
+    # Ensure that the strategy is valid
+    if strategy != "reflexion" and strategy != "simple":
+        raise ValueError(
+            f"Invalid strategy: given `{strategy}` but expected one of `reflexion` or `simple`")
+    if strategy == "reflexion" and (prev_func_impl is None or accumulated_feedback is None or accumulated_reflection is None):
+        raise ValueError(
+            f"Invalid arguments: given `strategy=reflexion` but `prev_func_impl`, `feedback`, or `self_reflection` is None")
+
+    # Build the accumulated context from the provided feedback and reflections
+    accumulated_context = "\n\n".join(
+        [f"[previous impl {i+1}]:\n{add_code_block(impl)}\n[unit test results from previous impl {i+1}]:\n{feedback}\n[reflection on previous impl {i+1}]:\n{reflection}" 
+         for i, (impl, feedback, reflection) in enumerate(zip(prev_func_impl, accumulated_feedback, accumulated_reflection))]
+    )
+
+    if model.is_chat:
+        if strategy == "reflexion":
+            # Constructing the message using a loop for accumulated context
+            messages = [
+                Message(role="system", content=f"{reflexion_chat_instruction}\n{code_block_instruction}"),
+                Message(role="user", content=reflexion_few_shot)
+            ]
+            
+            for impl, feedback, reflection in zip(prev_func_impl, accumulated_feedback, accumulated_reflection):
+                messages.append(Message(role="assistant", content=add_code_block(impl)))
+                messages.append(Message(role="user", content=f"[unit test results from previous impl]:\n{feedback}\n\n[reflection on previous impl]:\n{reflection}"))
+            
+            messages.append(Message(role="user", content=f"[improved impl]:\n{func_sig}"))
+            prompt = "\n".join([message.content for message in messages])
+            message = (f"{reflexion_few_shot}\n{accumulated_context}\n\n[improved impl]:\n{func_sig}")
+            print_messages(prompt, message)
+
+            func_bodies = model.generate_chat(messages=messages, num_comps=num_comps, temperature=temperature)
+        else:
+            system_prompt = f"{simple_chat_instruction}\n{code_block_instruction}"
+            print_messages(system_prompt, func_sig)
+            messages = [
+                Message(role="system", content=f"{simple_chat_instruction}\n{code_block_instruction}"),
+                Message(role="user", content=func_sig)
+            ]
+            func_bodies = model.generate_chat(messages=messages, num_comps=num_comps, temperature=temperature)
+    else:
+        if strategy == "reflexion":
+            prompt = f"{reflexion_completion_instruction}\n{accumulated_context}\n\n# improved implementation\n{func_sig}\n{code_block_instruction}"
+            func_bodies = model.generate(prompt, num_comps=num_comps, temperature=temperature)
+            print_messages(prompt, "")  
+        else:
+            prompt = f"{simple_completion_instruction}\n{func_sig}\n{code_block_instruction}"
+            func_bodies = model.generate(prompt, num_comps=num_comps, temperature=temperature)
+            print_messages(prompt, "")
+
+    if num_comps == 1:
+        assert isinstance(func_bodies, str)
+        func_body_str = parse_code_block(func_bodies)
+        print_generated_func_body(func_body_str)
+        return func_body_str
+
+    else:
+        func_bodies = [parse_code_block(func_body) for func_body in func_bodies]
+        print_generated_func_body("\n\n".join(func_bodies))
+        return func_bodies
+    
+
+def generic_generate_internal_tests(
+        func_sig: str,
+        model: ModelBase,
+        max_num_tests: int,
+        test_generation_few_shot: str,
+        test_generation_chat_instruction: str,
+        test_generation_completion_instruction: str,
+        parse_tests: Callable[[str], List[str]],
+        is_syntax_valid: Callable[[str], bool],
+        is_react: bool = False
+) -> List[str]:
+    """Generates tests for a function."""
+    if model.is_chat:
+        if is_react:
+            messages = [
+                Message(
+                    role="system",
+                    content=test_generation_chat_instruction,
+                ),
+                Message(
+                    role="user",
+                    content=f"{test_generation_few_shot}\n\n[func signature]:\n{func_sig}\n\n[think]:"
+                )
+            ]
+            output = model.generate_chat(messages=messages, max_tokens=1024)
+            print(f'React test generation output: {output}')
+        else:
+            messages = [
+                Message(
+                    role="system",
+                    content=test_generation_chat_instruction,
+                ),
+                Message(
+                    role="user",
+                    content=f"{test_generation_few_shot}\n\n[func signature]:\n{func_sig}\n\n[unit tests]:",
+                )
+            ]
+            output = model.generate_chat(messages=messages, max_tokens=1024)
+    else:
+        prompt = f'{test_generation_completion_instruction}\n\nfunc signature:\n{func_sig}\nunit tests:'
+        output = model.generate(prompt, max_tokens=1024)
+    all_tests = parse_tests(output)  # type: ignore
+    valid_tests = [test for test in all_tests if is_syntax_valid(test)]
+
+    # print(valid_tests)
+
+    return (valid_tests)
+
+
+def generic_generate_self_reflection(
+        func: str,
+        feedback: str,
+        model: ModelBase,
+        self_reflection_chat_instruction: str,
+        self_reflection_completion_instruction: str,
+        add_code_block: Callable[[str], str],
+        self_reflection_few_shot: Optional[str] = None,
+) -> str:
+    if model.is_chat:
+        if self_reflection_few_shot is not None:
+            messages = [
+                Message(
+                    role="system",
+                    content=self_reflection_chat_instruction,
+                ),
+                Message(
+                    role="user",
+                    content=f'{self_reflection_few_shot}\n\n[function impl]:\n{add_code_block(func)}\n\n[unit test results]:\n{feedback}\n\n[self-reflection]:',
+                )
+            ]
+            reflection = model.generate_chat(messages=messages)
+            print(f'|Self reflection output|: {reflection}')
+        else:
+            messages = [
+                Message(
+                    role="system",
+                    content=self_reflection_chat_instruction,
+                ),
+                Message(
+                    role="user",
+                    content=f'[function impl]:\n{add_code_block(func)}\n\n[unit test results]:\n{feedback}\n\n[self-reflection]:',
+                )
+            ]
+            reflection = model.generate_chat(messages=messages)
+    else:
+        reflection = model.generate(
+            f'{self_reflection_completion_instruction}\n{add_code_block(func)}\n\n{feedback}\n\nExplanation:')
+    return reflection  # type: ignore
+
+
+def sample_n_random(items: List[str], n: int) -> List[str]:
+    """Sample min(n, len(items)) random items from a list"""
+    assert n >= 0
+    if n >= len(items):
+        return items
+    return random.sample(items, n)
+
+def print_messages(system_message_text: str, user_message_text: str) -> None:
+    print(f"""{system_message_text}""")
+    print(f"""{user_message_text} \n""")
+
+def print_generated_func_body(func_body_str: str) -> None:
+    print(f"""|GENERATED FUNCTION BODY| \n
+```python\n{func_body_str} \n
+""")

generators/model.py

@@ -0,0 +1,120 @@
+from typing import List, Union, Optional, Literal
+import dataclasses
+
+from tenacity import (
+    retry,
+    stop_after_attempt,  # type: ignore
+    wait_random_exponential,  # type: ignore
+)
+import openai
+
+MessageRole = Literal["system", "user", "assistant"]
+
+
+@dataclasses.dataclass()
+class Message():
+    role: MessageRole
+    content: str
+
+
+def message_to_str(message: Message) -> str:
+    return f"{message.role}: {message.content}"
+
+
+def messages_to_str(messages: List[Message]) -> str:
+    return "\n".join([message_to_str(message) for message in messages])
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+def gpt_completion(
+        model: str,
+        prompt: str,
+        max_tokens: int = 1024,
+        stop_strs: Optional[List[str]] = None,
+        temperature: float = 0.0,
+        num_comps=1,
+) -> Union[List[str], str]:
+    response = openai.Completion.create(
+        model=model,
+        prompt=prompt,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        top_p=1,
+        frequency_penalty=0.0,
+        presence_penalty=0.0,
+        stop=stop_strs,
+        n=num_comps,
+    )
+    if num_comps == 1:
+        return response.choices[0].text  # type: ignore
+
+    return [choice.text for choice in response.choices]  # type: ignore
+
+
+@retry(wait=wait_random_exponential(min=1, max=180), stop=stop_after_attempt(6))
+def gpt_chat(
+    model: str,
+    messages: List,
+    max_tokens: int = 1024,
+    temperature: float = 0.0,
+    num_comps=1,
+) -> Union[List[str], str]:
+    try:
+        response = openai.ChatCompletion.create(
+            model=model,
+            messages=[dataclasses.asdict(message) for message in messages],
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=1,
+            frequency_penalty=0.0,
+            presence_penalty=0.0,
+            n=num_comps,
+        )
+        if num_comps == 1:
+            return response.choices[0].message.content  # type: ignore
+        return [choice.message.content for choice in response.choices]  # type: ignore
+
+    except Exception as e:
+        print(f"An error occurred while calling OpenAI: {e}")
+        raise
+
+class ModelBase():
+    def __init__(self, name: str):
+        self.name = name
+        self.is_chat = False
+
+    def __repr__(self) -> str:
+        return f'{self.name}'
+
+    def generate_chat(self, messages: List[Message], max_tokens: int = 1024, temperature: float = 0.2, num_comps: int = 1) -> Union[List[str], str]:
+        raise NotImplementedError
+
+    def generate(self, prompt: str, max_tokens: int = 1024, stop_strs: Optional[List[str]] = None, temperature: float = 0.0, num_comps=1) -> Union[List[str], str]:
+        raise NotImplementedError
+
+
+class GPTChat(ModelBase):
+    def __init__(self, model_name: str):
+        self.name = model_name
+        self.is_chat = True
+
+    def generate_chat(self, messages: List[Message], max_tokens: int = 1024, temperature: float = 0.2, num_comps: int = 1) -> Union[List[str], str]:
+        return gpt_chat(self.name, messages, max_tokens, temperature, num_comps)
+
+
+class GPT4(GPTChat):
+    def __init__(self):
+        super().__init__("gpt-4")
+
+
+class GPT35(GPTChat):
+    def __init__(self):
+        super().__init__("gpt-3.5-turbo")
+
+
+class GPTDavinci(ModelBase):
+    def __init__(self, model_name: str):
+        self.name = model_name
+
+    def generate(self, prompt: str, max_tokens: int = 1024, stop_strs: Optional[List[str]] = None, temperature: float = 0, num_comps=1) -> Union[List[str], str]:
+        return gpt_completion(self.name, prompt, max_tokens, stop_strs, temperature, num_comps)

generators/parse.py

@@ -0,0 +1,49 @@
+import re
+from typing import Optional
+
+
+def parse_code_block(string: str, lang: str) -> Optional[str]:
+    code_pattern = fr"```{lang}\n(.*?)\n```"
+    match = re.search(code_pattern, string, re.DOTALL)
+
+    if match:
+        return match.group(1)
+
+    generic_code_pattern = r"```\n(.*?)\n```"
+    match = re.search(generic_code_pattern, string, re.DOTALL)
+
+    if match:
+        return match.group(1)
+
+    return parse_first_func(string, lang)
+
+
+def parse_first_func(code: str, lang: str) -> Optional[str]:
+    assert lang == "python", "Only python is supported for now. TODO: Rust"
+    code_lines = code.split("\n")
+    def_i = -1
+    last_i = 0
+    got_return = False
+    for i, line in enumerate(code_lines):
+        if line.startswith("def "):
+            if def_i == -1:
+                def_i = i
+            else:
+                break
+        elif "return" in line and def_i != -1:
+            got_return = True
+        if line == "" and def_i != -1 and got_return:
+            last_i = i
+            break
+
+    if last_i == 0:
+        last_i = len(code_lines) - 1
+
+    if def_i == -1:
+        return None
+
+    return "\n".join(code_lines[def_i:last_i+1]).rstrip("[/PYTHON]")
+
+
+def add_code_block(string: str, lang: str) -> str:
+    return f"```{lang}\n{string}\n```"

generators/py_generate.py

@@ -0,0 +1,404 @@
+from generators.model import ModelBase, message_to_str
+from .generator_types import Generator
+from .generator_utils import generic_generate_func_impl, generic_generate_internal_tests, generic_generate_self_reflection, generate_with_accumulated_context
+
+from typing import Optional, List, Union
+import ast
+import re
+from .parse import parse_code_block, add_code_block
+
+PY_SIMPLE_COMPLETION_INSTRUCTION = "# Write the body of this function only."
+PY_REFLEXION_COMPLETION_INSTRUCTION = "You are a Python writing assistant. You will be given your past function implementation, a series of unit tests, and a hint to change the implementation appropriately. Write your full implementation (restate the function signature).\n\n-----"
+PY_SELF_REFLECTION_COMPLETION_INSTRUCTION = "You are a Python writing assistant. You will be given a function implementation and a series of unit tests. Your goal is to write a few sentences to explain why your implementation is wrong as indicated by the tests. You will need this as a hint when you try again later. Only provide the few sentence description in your answer, not the implementation.\n\n-----"
+USE_PYTHON_CODEBLOCK_INSTRUCTION = "Use a Python code block to write your response. For example:\n```python\nprint('Hello world!')\n```"
+
+PY_SIMPLE_CHAT_INSTRUCTION = "You are an AI that only responds with python code, NOT ENGLISH. You will be given a function signature and its docstring by the user. Write your full implementation (restate the function signature)."
+PY_SIMPLE_CHAT_INSTRUCTION_V2 = "You are an AI that only responds with only python code. You will be given a function signature and its docstring by the user. Write your full implementation (restate the function signature)."
+PY_REFLEXION_CHAT_INSTRUCTION = "You are an AI Python assistant. You will be given your past function implementation, a series of unit tests, and a hint to change the implementation appropriately. Write your full implementation (restate the function signature)."
+PY_REFLEXION_CHAT_INSTRUCTION_V2 = "You are an AI Python assistant. You will be given your previous implementation of a function, a series of unit tests results, and your self-reflection on your previous implementation. Write your full implementation (restate the function signature)."
+PY_REFLEXION_FEW_SHOT_ADD = '''Example 1:
+[previous impl]:
+```python
+def add(a: int, b: int) -> int:
+    """
+    Given integers a and b, return the total value of a and b.
+    """
+    return a - b
+```
+
+[unit test results from previous impl]:
+Tested passed:
+
+Tests failed:
+assert add(1, 2) == 3 # output: -1
+assert add(1, 2) == 4 # output: -1
+
+[reflection on previous impl]:
+The implementation failed the test cases where the input integers are 1 and 2. The issue arises because the code does not add the two integers together, but instead subtracts the second integer from the first. To fix this issue, we should change the operator from `-` to `+` in the return statement. This will ensure that the function returns the correct output for the given input.
+
+[improved impl]:
+```python
+def add(a: int, b: int) -> int:
+    """
+    Given integers a and b, return the total value of a and b.
+    """
+    return a + b
+```
+'''
+
+PY_REFLEXION_FEW_SHOT = '''Example 1:
+[previous impl]:
+```python
+from typing import *
+def fullJustify(words: List[str], maxWidth: int) -> List[str]:
+    """
+    Given an array of words and a width maxWidth, format the text such that each line has exactly maxWidth characters and is fully (left and right) justified.
+    You should pack your words in a greedy approach; that is, pack as many words as you can in each line. Pad extra spaces `' '` when necessary so that each line has exactly maxWidth characters.
+    Extra spaces between words should be distributed as evenly as possible. If the number of spaces on a line do not divide evenly between words, the empty slots on the left will be assigned more spaces than the slots on the right.
+    For the last line of text, it should be left justified and no extra space is inserted between words.
+    Note:
+    A word is defined as a character sequence consisting of non-space characters only.
+    Each word's length is guaranteed to be greater than 0 and not exceed maxWidth.
+    The input array `words` contains at least one word.
+    """
+    res = []
+    cur_line = []
+    cur_len = 0
+
+    for word in words:
+        if cur_len + len(word) + len(cur_line) > maxWidth:
+            if len(cur_line) == 1:
+                res.append(cur_line[0] + ' ' * (maxWidth - cur_len))
+            else:
+                spaces = maxWidth - cur_len
+                space_between = spaces // (len(cur_line) - 1)
+                extra_spaces = spaces % (len(cur_line) - 1)
+                line = ''
+                for i, w in enumerate(cur_line[:-1]):
+                    line += w + ' ' * (space_between + (i < extra_spaces))
+                line += cur_line[-1]
+                res.append(line)
+            cur_line = []
+            cur_len = 0
+        cur_line.append(word)
+        cur_len += len(word)
+
+    last_line = ' '.join(cur_line)
+    last_line += ' ' * (maxWidth - len(last_line))
+    res.append(last_line)
+
+    return res
+```
+
+[unit test results from previous impl]:
+Tested passed:
+
+Tests failed:
+assert fullJustify([], 10) == [] # output: ['          ']
+assert fullJustify([], 0) == [] # output: ['']
+
+[reflection on previous impl]:
+The implementation failed the test cases where the input list of words is empty. The issue arises because the code does not handle the case where there are no words to process. As a result, it still appends a line with spaces to the result list, even when there are no words. To fix this issue, we should add a condition at the beginning of the function to check if the input list is empty, and return an empty list if it is. This will ensure that the function returns the correct output for empty input lists.
+
+[improved impl]:
+```python
+from typing import *
+def fullJustify(words: List[str], maxWidth: int) -> List[str]:
+    """
+    Given an array of words and a width maxWidth, format the text such that each line has exactly maxWidth characters and is fully (left and right) justified.
+    You should pack your words in a greedy approach; that is, pack as many words as you can in each line. Pad extra spaces `' '` when necessary so that each line has exactly maxWidth characters.
+    Extra spaces between words should be distributed as evenly as possible. If the number of spaces on a line do not divide evenly between words, the empty slots on the left will be assigned more spaces than the slots on the right.
+    For the last line of text, it should be left justified and no extra space is inserted between words.
+    Note:
+    A word is defined as a character sequence consisting of non-space characters only.
+    Each word's length is guaranteed to be greater than 0 and not exceed maxWidth.
+    The input array `words` contains at least one word.
+    """
+    if not words:
+        return []
+
+    res = []
+    cur_line = []
+    cur_len = 0
+
+    for word in words:
+        if cur_len + len(word) + len(cur_line) > maxWidth:
+            if len(cur_line) == 1:
+                res.append(cur_line[0] + ' ' * (maxWidth - cur_len))
+            else:
+                spaces = maxWidth - cur_len
+                space_between = spaces // (len(cur_line) - 1)
+                extra_spaces = spaces % (len(cur_line) - 1)
+                line = ''
+                for i, w in enumerate(cur_line[:-1]):
+                    line += w + ' ' * (space_between + (i < extra_spaces))
+                line += cur_line[-1]
+                res.append(line)
+            cur_line = []
+            cur_len = 0
+        cur_line.append(word)
+        cur_len += len(word)
+
+    last_line = ' '.join(cur_line)
+    last_line += ' ' * (maxWidth - len(last_line))
+    res.append(last_line)
+
+    return res
+```
+END EXAMPLES
+
+'''
+PY_SELF_REFLECTION_CHAT_INSTRUCTION = "You are a Python programming assistant. You will be given a function implementation and a series of unit tests. Your goal is to write a few sentences to explain why your implementation is wrong as indicated by the tests. You will need this as a hint when you try again later. Only provide the few sentence description in your answer, not the implementation."
+PY_SELF_REFLECTION_CHAT_INSTRUCTION_V2 = "You are a Python programming assistant. You will be given a function implementation and a series of unit test results. Your goal is to write a few sentences to explain why your implementation is wrong as indicated by the tests. You will need this as guidance when you try again later. Only provide the few sentence description in your answer, not the implementation. You will be given a few examples by the user."
+PY_SELF_REFLECTION_FEW_SHOT = """Example 1:
+[function impl]:
+```python
+def longest_subarray_with_sum_limit(nums: List[int], target: int) -> List[int]:
+    n = len(nums)
+    left, right = 0, 0
+    max_length = 0
+    current_sum = 0
+    result = []
+    while right < n:
+        current_sum += nums[right]
+        while current_sum > target:
+            current_sum -= nums[left]
+            left += 1
+        if right - left + 1 >= max_length:
+            max_length = right - left + 1
+            result = nums[left:right+1]
+        right += 1
+    return result
+```
+[unit test results]:
+Tests passing:
+assert longest_subarray_with_sum_limit([1, 2, 3, 4, 5], 8) == [1, 2, 3]
+assert longest_subarray_with_sum_limit([1, 2, 3, 4, 5], 15) == [1, 2, 3, 4, 5]
+assert longest_subarray_with_sum_limit([1, -1, 2, -2, 3, -3], 2) == [1, -1, 2, -2, 3]
+assert longest_subarray_with_sum_limit([], 10) == []
+assert longest_subarray_with_sum_limit([], 0) == []
+assert longest_subarray_with_sum_limit([], -5) == []  
+Tests failing:
+assert longest_subarray_with_sum_limit([5, 6, 7, 8, 9], 4) == [] # output: [5]
+[self-reflection]:
+The implementation failed the where no subarray fulfills the condition. The issue in the implementation is due to the use of >= instead of > in the condition to update the result. Because of this, it returns a subarray even when the sum is greater than the target, as it still updates the result when the current subarray length is equal to the previous longest subarray length. To overcome this error, we should change the condition to only update the result when the current subarray length is strictly greater than the previous longest subarray length. This can be done by replacing >= with > in the condition.
+
+Example 2:
+[function impl]:
+```python
+def longest_subarray_with_sum_limit(nums: List[int], target: int) -> List[int]:
+    n = len(nums)
+    left, right = 0, 0
+    max_length = 0
+    current_sum = 0
+    result = []
+    while current_sum + nums[right] <= target:
+        current_sum += nums[right]
+        right += 1
+    while right < n:
+        current_sum += nums[right]
+        while current_sum > target:
+            current_sum -= nums[left]
+            left += 1
+        if right - left + 1 > max_length:
+            max_length = right - left + 1
+            result = nums[left:right+1]
+        right += 1
+    return result
+```
+[unit test results]:
+Tests passing:
+assert longest_subarray_with_sum_limit([], 10) == []
+assert longest_subarray_with_sum_limit([], 0) == []
+assert longest_subarray_with_sum_limit([], -5) == []
+Tests failing:
+assert longest_subarray_with_sum_limit([1, 2, 3, 4, 5], 8) == [1, 2, 3] # output: list index out of range
+assert longest_subarray_with_sum_limit([1, 2, 3, 4, 5], 15) == [1, 2, 3, 4, 5] # output: list index out of range
+assert longest_subarray_with_sum_limit([5, 6, 7, 8, 9], 4) == [] # output: list index out of range
+assert longest_subarray_with_sum_limit([1, -1, 2, -2, 3, -3], 2) == [1, -1, 2, -2, 3] # output: list index out of range
+[self-reflection]:
+The implementation failed 4 out of the 7 test cases due to an IndexError. The issue stems from the while loop while current_sum + nums[right] <= target:, which directly accesses nums[right] without checking if right is within the bounds of the list. This results in a runtime error when right goes beyond the list length. To overcome this error, we need to add a bounds check for the right variable in the mentioned while loop. We can modify the loop condition to while right < len(nums) and current_sum + nums[right] <= target:. This change will ensure that we only access elements within the bounds of the list, thus avoiding the IndexError.
+END OF EXAMPLES
+"""
+
+PY_TEST_GENERATION_FEW_SHOT = """Examples:
+func signature:
+def add3Numbers(x, y, z):
+    \"\"\" Add three numbers together.
+    This function takes three numbers as input and returns the sum of the three numbers.
+    \"\"\"
+unit tests:
+assert add3Numbers(1, 2, 3) == 6
+assert add3Numbers(-1, 2, 3) == 4
+assert add3Numbers(1, -2, 3) == 2
+assert add3Numbers(1, 2, -3) == 0
+assert add3Numbers(-3, -2, -1) == -6
+assert add3Numbers(0, 0, 0) == 0
+"""
+
+PY_TEST_GENERATION_COMPLETION_INSTRUCTION = f"""You are an AI coding assistant that can write unique, diverse, and intuitive unit tests for functions given the signature and docstring. Call your function answer().
+
+{PY_TEST_GENERATION_FEW_SHOT}"""
+
+PY_TEST_GENERATION_CHAT_INSTRUCTION = """You are an AI coding assistant that can write unique, diverse, and intuitive unit tests for functions given the signature and docstring. Call your function answer()."""
+
+
+class PyGenerator(Generator):
+    def self_reflection(self, func: str, feedback: str, model: ModelBase) -> str:
+        return generic_generate_self_reflection(
+            func=func,
+            feedback=feedback,
+            model=model,
+            self_reflection_chat_instruction=PY_SELF_REFLECTION_CHAT_INSTRUCTION,
+            self_reflection_completion_instruction=PY_SELF_REFLECTION_COMPLETION_INSTRUCTION,
+            add_code_block=lambda x: add_code_block(x, "python"),
+            self_reflection_few_shot=PY_SELF_REFLECTION_FEW_SHOT
+        )
+
+    def func_impl(
+        self,
+        func_sig: str,
+        model: ModelBase,
+        strategy: str,
+        prev_func_impl: Optional[str] = None,
+        feedback: Optional[str] = None,
+        self_reflection: Optional[str] = None,
+        num_comps: int = 1,
+        temperature: float = 0.8,
+        acc_feedback: Optional[str] = None,
+        acc_reflection: Optional[str] = None,
+    ) -> Union[str, List[str]]:
+        if strategy == "mcts":
+            return generate_with_accumulated_context(
+                func_sig=func_sig,
+                model=model,
+                strategy="reflexion",
+                prev_func_impl=prev_func_impl,
+                accumulated_feedback=acc_feedback,
+                accumulated_reflection=acc_reflection,
+                num_comps=num_comps,
+                temperature=temperature,
+                reflexion_chat_instruction=PY_REFLEXION_CHAT_INSTRUCTION,
+                reflexion_few_shot=PY_REFLEXION_FEW_SHOT_ADD,
+                simple_chat_instruction=PY_SIMPLE_CHAT_INSTRUCTION,
+                reflexion_completion_instruction=PY_REFLEXION_COMPLETION_INSTRUCTION,
+                simple_completion_instruction=PY_SIMPLE_COMPLETION_INSTRUCTION,
+                code_block_instruction=USE_PYTHON_CODEBLOCK_INSTRUCTION,
+                parse_code_block=lambda x: parse_code_block(x, "python"),
+                add_code_block=lambda x: add_code_block(x, "python"),
+            )
+        else:
+            return generic_generate_func_impl(
+                func_sig=func_sig,
+                model=model,
+                strategy=strategy,
+                prev_func_impl=prev_func_impl,
+                feedback=feedback,
+                self_reflection=self_reflection,
+                num_comps=num_comps,
+                temperature=temperature,
+                reflexion_chat_instruction=PY_REFLEXION_CHAT_INSTRUCTION,
+                reflexion_few_shot=PY_REFLEXION_FEW_SHOT_ADD,
+                simple_chat_instruction=PY_SIMPLE_CHAT_INSTRUCTION,
+                reflexion_completion_instruction=PY_REFLEXION_COMPLETION_INSTRUCTION,
+                simple_completion_instruction=PY_SIMPLE_COMPLETION_INSTRUCTION,
+                code_block_instruction=USE_PYTHON_CODEBLOCK_INSTRUCTION,
+                parse_code_block=lambda x: parse_code_block(x, "python"),
+                add_code_block=lambda x: add_code_block(x, "python"),
+            )
+
+    def internal_tests(self, func_sig: str, model: ModelBase, max_num_tests: int = 4) -> List[str]:
+        def parse_tests(tests: str) -> List[str]:
+            return [test.strip() for test in tests.splitlines() if "assert" in test]
+        """
+        Generates tests for a function.
+        """
+        return generic_generate_internal_tests(
+            func_sig=func_sig,
+            model=model,
+            max_num_tests=max_num_tests,
+            test_generation_few_shot=PY_TEST_GENERATION_FEW_SHOT,
+            test_generation_chat_instruction=PY_TEST_GENERATION_CHAT_INSTRUCTION,
+            test_generation_completion_instruction=PY_TEST_GENERATION_COMPLETION_INSTRUCTION,
+            parse_tests=parse_tests,
+            is_syntax_valid=py_is_syntax_valid,
+        )
+
+
+DUMMY_FUNC_SIG = "def func():"
+DUMMY_FUNC_CALL = "func()"
+
+
+def handle_first_line_indent(func_body: str) -> str:
+    if func_body.startswith("    "):
+        return func_body
+    split = func_body.splitlines()
+    return f"    {split[0]}\n" + "\n".join(split[1:])
+
+
+def handle_entire_body_indent(func_body: str) -> str:
+    split = func_body.splitlines()
+    res = "\n".join(["    " + line for line in split])
+    return res
+
+
+def fix_turbo_response(func_body: str) -> str:
+    return fix_markdown(remove_unindented_signatures(func_body))
+
+
+def fix_markdown(func_body: str) -> str:
+    return re.sub("`{3}", "", func_body)
+
+
+def remove_unindented_signatures(code: str) -> str:
+    regex = r"^def\s+\w+\s*\("
+
+    before_signature = []
+    after_signature = []
+    signature_found = False
+
+    for line in code.split("\n"):
+        if re.match(regex, line):
+            signature_found = True
+            continue
+
+        if signature_found:
+            after_signature.append(line)
+        else:
+            if not line.startswith("    ") and line.strip():
+                line = "    " + line
+            before_signature.append(line)
+
+    return "\n".join(before_signature + after_signature)
+
+
+def py_fix_indentation(func_body: str) -> str:
+    func_body = fix_turbo_response(func_body)
+    """
+    3 cases:
+        1. good syntax
+        2. first line not good
+        3. entire body not good
+    """
+    def parse_indent_rec(f_body: str, cur_state: int) -> str:
+        f_body = fix_markdown(f_body)
+        if cur_state > 1:
+            return f_body
+        code = f'{DUMMY_FUNC_SIG}\n{f_body}\n{DUMMY_FUNC_CALL}'
+        try:
+            exec(code)
+            return f_body
+        except (IndentationError, SyntaxError):
+            p_func = handle_first_line_indent if cur_state == 0 else handle_entire_body_indent
+            return parse_indent_rec(p_func(func_body), cur_state + 1)
+        except Exception:
+            return f_body
+    return parse_indent_rec(func_body, 0)
+
+
+def py_is_syntax_valid(code: str) -> bool:
+    try:
+        ast.parse(code)
+        return True
+    except Exception:
+        return False

lats/lats.py

@@ -0,0 +1,233 @@
+from utils import enumerate_resume, make_printv, write_jsonl, resume_success_count
+from executors import executor_factory
+from generators import generator_factory, model_factory
+from typing import List, Dict, Any
+import math
+from typing import Tuple
+import sys
+import random
+
+sys.set_int_max_str_digits(100000)  # Increase the limit to 10000 digits
+
+react_prompt_header = "Here are some previous solutions and the corresponding test results.\n"
+react_prompt_starter = "\n\nYour solution:\n"
+extra_header = "\n\nName the function answer()"
+
+class Node:
+    def __init__(self, solution: str, parent=None, context="", depth=0):
+        self.solution = solution
+        self.parent = parent
+        self.children = []
+        self.value = 0
+        self.visits = 0
+        self.context = ""
+        self.depth = depth
+        self.reflection = ""
+        self.test_feedback = ""
+
+    def uct(self, exploration_weight=1.0):
+        if self.visits == 0:
+            #return float('inf')
+            return self.value
+        return (self.value / self.visits) + exploration_weight * math.sqrt(math.log(self.parent.visits) / self.visits)
+
+    def best_child(self):
+        if not self.children:  # Check if children list is empty
+            return None
+        return max(self.children, key=lambda child: child.uct())
+
+    def best_child_value(self):
+        if not self.children:  # Check if children list is empty
+            return None
+        return max(self.children, key=lambda child: child.value)
+
+    def update(self, reward: float):
+        self.visits += 1
+        self.value += reward
+    
+
+def prune_context_blocks(context: str, max_length: int) -> str:
+    """Prune the context to fit within the specified max_length by removing entire blocks of content using 'trial' as a delimiter."""
+    if len(context) <= max_length:
+        return context
+    
+    # Split by the block delimiter "trial".
+    blocks = context.split('Previous Trial')
+    
+    # Remove the earliest blocks until the context fits within max_length.
+    while len('trial'.join(blocks)) > max_length and blocks:
+        blocks.pop(0)
+    
+    return 'trial'.join(blocks)
+
+def gather_context_from_tree(node: Node) -> Tuple[List[str], List[str]]:
+    """
+    Given a node, walk up its tree and gather the feedback and reflections 
+    from each parent node until the root is reached.
+
+    Args:
+        node (Node): The node to start gathering context from.
+
+    Returns:
+        Tuple[List[str], List[str]]: Two lists containing the accumulated feedback and reflections.
+    """
+    accumulated_feedback = []
+    accumulated_reflection = []
+    num_nodes = 0
+
+    while node and num_nodes < 2:
+        num_nodes += 1
+        if node.test_feedback:
+            accumulated_feedback.append(node.test_feedback)
+        if node.reflection:
+            accumulated_reflection.append(node.reflection)
+        node = node.parent
+
+    # Reverse the lists so that the context from the earliest nodes is first
+    return accumulated_feedback[::-1], accumulated_reflection[::-1]
+
+def sample_n_random(items: List[str], n: int) -> List[str]:
+    """Sample min(n, len(items)) random items from a list"""
+    assert n >= 0
+    if n >= len(items):
+        return items
+    return random.sample(items, n)
+
+def run_lats(
+    model_name: str,
+    language: str,
+    max_iters: int,
+    verbose: bool,
+    instruction: str = "Write some code to print Hello World in Python",
+    n_samples: int = 3,
+    depth: int = 5,
+) -> None:
+    exe = executor_factory(language)
+    gen = generator_factory(language)
+    model = model_factory(model_name)
+
+
+    num_success = 0  # Counter for successful solutions
+    cur_func_impl = None
+
+    item = {}
+
+    #for idx, item in enumerate(dataset):
+    
+    tests = gen.internal_tests(instruction + extra_header, model, 1)
+    tests_i = sample_n_random(tests, 1)
+
+    while cur_func_impl is None:
+        cur_func_impl = gen.func_impl(instruction + extra_header, model, "simple")
+    root = Node(cur_func_impl) # initial solution (for pass@1 metric)
+    
+    # Lists for logging
+    reflections = []
+    implementations = []
+    test_feedback = []
+    is_solved = False
+
+    # first attempt
+    
+    implementations.append(cur_func_impl)
+    assert isinstance(cur_func_impl, str)
+    is_passing, feedback, _ = exe.execute(cur_func_impl, tests_i)
+    test_feedback.append(feedback)
+
+    # if solved, exit early
+    if is_passing:
+        num_success += 1
+        return cur_func_impl # GET SOLUTION
+    
+    reflection = gen.self_reflection(cur_func_impl, feedback, model)
+    reflections += [reflection]
+    root.test_feedback = feedback
+    root.reflection = reflection
+    max_iters = int(max_iters)
+    for cur_iter in range(max_iters):
+        # Selection
+        tests_i = sample_n_random(tests, 1)
+
+        node = root
+        trajectory = {
+            'solutions': [],
+            'feedbacks': []
+        }
+
+        while node.children:
+            node = node.best_child()
+            trajectory['solutions'].append(node.solution)
+        
+        # Expansion
+        for _ in range(n_samples):
+            new_solution = None
+            strategy = "mcts"
+            prev_func_impl = node.solution
+            feedback = node.test_feedback
+            reflection = node.reflection
+            acc_feedback, acc_reflection = gather_context_from_tree(node)
+            
+            while new_solution is None:
+                new_solution = gen.func_impl(
+                    func_sig=instruction+extra_header,
+                    model=model,
+                    strategy=strategy,
+                    prev_func_impl=prev_func_impl,
+                    feedback=feedback,
+                    self_reflection=reflection,
+                    acc_feedback = acc_feedback,
+                    acc_reflection = acc_reflection
+                )
+
+            combined_context = "\nPrevious Trial\n\n" + new_solution
+
+            child = Node(new_solution, parent=node, context=combined_context, depth=node.depth + 1)
+            node.children.append(child)
+
+            # Simulation
+            reward_real = 0
+            for child in node.children:
+                is_passing_internal, feedback_internal, _ = exe.execute(child.solution, tests_i)
+                if not is_passing_internal:
+                    reflection = gen.self_reflection(child.solution, feedback_internal, model)
+                    reflections.append(reflection)
+                    child.reflection = reflection
+                    child.test_feedback = feedback_internal
+                    child.context += "\n\nPrevious Trial\n\n" + child.solution + "\n\nTest results: \n" + feedback_internal + "\n\nSelf-reflection: " + reflection
+                else:
+                    child.context += "\n\nPrevious Trial\n\n" + child.solution + "\n\nTest results: \n" + feedback_internal
+                    child.reflection = ""
+                    child.test_feedback = feedback_internal
+
+                if "Tested passed:" in feedback_internal:
+                    # Split at "Tests failed:" and get the part before it (which contains the passed tests)
+                    passed_section = feedback_internal.split("Tests failed:")[0]
+                    # Split at "Tested passed:" and get the part after it, then count the non-empty lines
+                    reward_internal = len([line for line in passed_section.split("Tested passed:")[1].splitlines() if line.strip() != ''])
+                    reward_internal = reward_internal / len(tests_i)
+                else:
+                    reward_internal = 0
+                if is_passing_internal or cur_iter == max_iters - 1:
+                    item["solution"] = child.solution
+                    break
+
+            if is_solved:
+                break
+            
+            reward = reward_internal + reward_real
+            child.update(reward)
+
+            # Backpropagation
+            temp = child
+            while temp.parent:
+                temp = temp.parent
+                temp.update(reward)
+    
+    # Choose the best solution after all iterations
+    if is_solved:
+        best_solution = item["solution"]
+    else:
+        best_solution = root.best_child_value().solution
+        item["solution"] = best_solution
+
+    return best_solution

lats/lats_main.py

@@ -0,0 +1,78 @@
+import os
+import argparse
+
+from lats import run_lats
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run_name", type=str, help="The name of the run")
+    parser.add_argument("--root_dir", type=str,
+                        help="The root logging directory", default="root")
+    parser.add_argument("--dataset_path", type=str,
+                        help="The path to the benchmark dataset", default="root")
+    parser.add_argument("--strategy", type=str,
+                        help="Strategy: `simple`, `reflexion`")
+    parser.add_argument("--language", type=str, help="Strategy: `py` or `rs`")
+    parser.add_argument(
+        "--model", type=str, help="OpenAI models only for now. For best results, use GPT-4")
+    parser.add_argument("--pass_at_k", type=int,
+                        help="Pass@k metric", default=1)
+    parser.add_argument("--max_iters", type=int,
+                        help="The maximum number of self-improvement iterations", default=10)
+    parser.add_argument("--expansion_factor", type=int,
+                        help="The expansion factor for the reflexion UCS and A* strategy", default=3)
+    parser.add_argument("--verbose", action='store_true',
+                        help="To print live logs")
+    parser.add_argument("--instruction", type=str,
+                        help="text string", default="")
+    parser.add_argument("--n_samples", type=int,
+                        help="The number of nodes added during expansion", default=3)
+    parser.add_argument("--depth", type=int,
+                        help="Tree depth", default=5)
+
+    # TODO: implement this
+    # parser.add_argument("--is_resume", action='store_true', help="To resume run")
+    # parser.add_argument("--resume_dir", type=str, help="If resume, the logging directory", default="")
+    args = parser.parse_args()
+    return args
+
+
+def strategy_factory(strategy: str):
+    def kwargs_wrapper_gen(func, delete_keys=[]):
+        def kwargs_wrapper(**kwargs):
+            for key in delete_keys:
+                del kwargs[key]
+            return func(**kwargs)
+        return kwargs_wrapper
+
+    return kwargs_wrapper_gen(run_lats, delete_keys=[])
+    
+
+def lats_main(args):
+
+    # check if the strategy is valid
+    run_strategy = strategy_factory(args.strategy)
+
+    # start the run
+    # evaluate with pass@k
+    x = run_strategy(
+        model_name=args.model,
+        language=args.language,
+        max_iters=args.max_iters,
+        verbose=args.verbose,
+        instruction=args.instruction,
+        n_samples=args.n_samples,
+        depth=args.depth
+    )
+
+    return x
+
+
+
+def main(args):
+    lats_main(args)
+
+if __name__ == "__main__":
+    args = get_args()
+    main(args)

lats/requirements.txt

@@ -0,0 +1,9 @@
+jsonlines==3.1.0
+openai==0.27.0
+datasets==2.7.0
+tenacity==8.1.0
+astunparse==1.6.3
+torch
+xformers
+transformers
+accelerate

lats/utils.py

@@ -0,0 +1,73 @@
+import os
+import gzip
+import json
+import openai
+import jsonlines
+
+from typing import List
+
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+def make_printv(verbose: bool):
+    def print_v(*args, **kwargs):
+        if verbose:
+            kwargs["flush"] = True
+            print(*args, **kwargs)
+        else:
+            pass
+    return print_v
+
+
+def read_jsonl(path: str) -> List[dict]:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"File `{path}` does not exist.")
+    elif not path.endswith(".jsonl"):
+        raise ValueError(f"File `{path}` is not a jsonl file.")
+    items = []
+    with jsonlines.open(path) as reader:
+        for item in reader:
+            items += [item]
+    return items
+
+
+def write_jsonl(path: str, data: List[dict], append: bool = False):
+    with jsonlines.open(path, mode='a' if append else 'w') as writer:
+        for item in data:
+            writer.write(item)
+
+
+def read_jsonl_gz(path: str) -> List[dict]:
+    if not path.endswith(".jsonl.gz"):
+        raise ValueError(f"File `{path}` is not a jsonl.gz file.")
+    with gzip.open(path, "rt") as f:
+        data = [json.loads(line) for line in f]
+    return data
+
+
+# generator that returns the item and the index in the dataset.
+# if the results_path exists, it will skip all items that have been processed
+# before.
+def enumerate_resume(dataset, results_path):
+    if not os.path.exists(results_path):
+        for i, item in enumerate(dataset):
+            yield i, item
+    else:
+        count = 0
+        with jsonlines.open(results_path) as reader:
+            for item in reader:
+                count += 1
+
+        for i, item in enumerate(dataset):
+            # skip items that have been processed before
+            if i < count:
+                continue
+            yield i, item
+
+
+def resume_success_count(dataset) -> int:
+    count = 0
+    for item in dataset:
+        if "is_solved" in item and item["is_solved"]:
+            count += 1
+    return count
+

requirements.txt

@@ -0,0 +1,9 @@
+jsonlines==3.1.0
+openai==0.27.0
+datasets==2.7.0
+tenacity==8.1.0
+astunparse==1.6.3
+torch
+transformers
+accelerate
+xformers