ronantakizawa/codereview-bench huggingface dataset:
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/db.py (Python)

Code before:
from dataclasses import dataclass
import os
from pathlib import Path


class DB:
    """A simple key-value store, where keys are filenames and values are file contents."""

    def __init__(self, path):
        self.path = Path(path).absolute()
        os.makedirs(self.path, exist_ok=True)

    def __getitem__(self, key):
        try:
            with open(self.path / key, encoding='utf-8') as f:
                return f.read()
        except FileNotFoundError:
            raise KeyError(key)

    def __setitem__(self, key, val):
        Path(self.path / key).absolute().parent.mkdir(parents=True, exist_ok=True)

        with open(self.path / key, 'w', encoding='utf-8') as f:
            f.write(str(val))

    def __contains__(self, key):
        return (self.path / key).exists()


@dataclass
class DBs:
    """A dataclass for all dbs"""

    memory: DB
    logs: DB
    identity: DB
    input: DB
    workspace: DB


Review comment (bug):
```suggestion
            f.write(val)
```

I think we should only allow str as input here, it's probably something wrong if it's not a string

Code after:
from dataclasses import dataclass
from pathlib import Path


# This class represents a simple database that stores its data as files in a directory.
class DB:
    """A simple key-value store, where keys are filenames and values are file contents."""

    def __init__(self, path):
        self.path = Path(path).absolute()

        self.path.mkdir(parents=True, exist_ok=True)

    def __contains__(self, key):
        return (self.path / key).is_file()

    def __getitem__(self, key):
        full_path = self.path / key

        if not full_path.is_file():
            raise KeyError(key)
        with full_path.open("r", encoding="utf-8") as f:
            return f.read()

    def __setitem__(self, key, val):
        full_path = self.path / key
        full_path.parent.mkdir(parents=True, exist_ok=True)

        if isinstance(val, str):
            full_path.write_text(val, encoding="utf-8")
        else:
            # If val is neither a string nor bytes, raise an error.
            raise TypeError("val must be either a str or bytes")


# dataclass for all dbs:
@dataclass
class DBs:
    memory: DB
    logs: DB
    preprompts: DB
    input: DB
    workspace: DB
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/steps.py (Python)

Code before:
    print()

    regex = r"```\S*\n(.+?)```"
    matches = re.finditer(regex, messages[-1]["content"], re.DOTALL)
    dbs.workspace["run.sh"] = "\n".join(match.group(1) for match in matches)
    return messages


def use_feedback(ai: AI, dbs: DBs):
    messages = [
        ai.fsystem(setup_sys_prompt(dbs)),
        ai.fuser(f"Instructions: {dbs.input['main_prompt']}"),
        ai.fassistant(dbs.workspace["all_output.txt"]),
        ai.fsystem(dbs.preprompts["use_feedback"]),
    ]
    messages = ai.next(messages, dbs.input["feedback"])
    to_files(messages[-1]["content"], dbs.workspace)
    return messages


def fix_code(ai: AI, dbs: DBs):
    code_ouput = json.loads(dbs.logs[gen_code.__name__])[-1]["content"]
    messages = [
        ai.fsystem(setup_sys_prompt(dbs)),
        ai.fuser(f"Instructions: {dbs.input['main_prompt']}"),
        ai.fuser(code_ouput),
        ai.fsystem(dbs.preprompts["fix_code"]),
    ]
    messages = ai.next(messages, "Please fix any errors in the code above.")
    to_files(messages[-1]["content"], dbs.workspace)
    return messages


class Config(str, Enum):
    DEFAULT = "default"
    BENCHMARK = "benchmark"
    SIMPLE = "simple"
    TDD = "tdd"
    TDD_PLUS = "tdd+"
    CLARIFY = "clarify"
    RESPEC = "respec"
    EXECUTE_ONLY = "execute_only"
    USE_FEEDBACK = "use_feedback"


# Different configs of what steps to run
STEPS = {
    Config.DEFAULT: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,

Review comment (suggestion):
```suggestion
        ai.fuser(code_output),
```

Code after:
    print()

    regex = r"```\S*\n(.+?)```"
    matches = re.finditer(regex, messages[-1]["content"], re.DOTALL)
    dbs.workspace["run.sh"] = "\n".join(match.group(1) for match in matches)
    return messages


def use_feedback(ai: AI, dbs: DBs):
    messages = [
        ai.fsystem(setup_sys_prompt(dbs)),
        ai.fuser(f"Instructions: {dbs.input['main_prompt']}"),
        ai.fassistant(dbs.workspace["all_output.txt"]),
        ai.fsystem(dbs.preprompts["use_feedback"]),
    ]
    messages = ai.next(messages, dbs.input["feedback"])
    to_files(messages[-1]["content"], dbs.workspace)
    return messages


def fix_code(ai: AI, dbs: DBs):
    code_output = json.loads(dbs.logs[gen_code.__name__])[-1]["content"]
    messages = [
        ai.fsystem(setup_sys_prompt(dbs)),
        ai.fuser(f"Instructions: {dbs.input['main_prompt']}"),
        ai.fuser(code_output),
        ai.fsystem(dbs.preprompts["fix_code"]),
    ]
    messages = ai.next(messages, "Please fix any errors in the code above.")
    to_files(messages[-1]["content"], dbs.workspace)
    return messages


class Config(str, Enum):
    DEFAULT = "default"
    BENCHMARK = "benchmark"
    SIMPLE = "simple"
    TDD = "tdd"
    TDD_PLUS = "tdd+"
    CLARIFY = "clarify"
    RESPEC = "respec"
    EXECUTE_ONLY = "execute_only"
    USE_FEEDBACK = "use_feedback"


# Different configs of what steps to run
STEPS = {
    Config.DEFAULT: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/main.py (Python)

Code before:

    model = fallback_model(model)
    ai = AI(
        model=model,
        temperature=temperature,
    )

    input_path = Path(project_path).absolute()
    memory_path = input_path / f"{run_prefix}memory"
    workspace_path = input_path / f"{run_prefix}workspace"
    archive_path = input_path / f"{run_prefix}archive"

    initial_run = not os.path.exists(memory_path) and not os.path.exists(workspace_path)
    dbs = DBs(
        memory=DB(memory_path),
        logs=DB(memory_path / "logs"),
        input=DB(input_path),
        workspace=DB(workspace_path),
        preprompts=DB(Path(__file__).parent / "preprompts"),
        archive=None if initial_run else DB(archive_path),
    )

    steps = STEPS[steps_config]
    for step in steps:
        messages = step(ai, dbs)
        # TODO: Don't dump if step returns None
        # human_review, archive don't need to be dumped
        dbs.logs[step.__name__] = json.dumps(messages)

    collect_learnings(model, temperature, steps, dbs)


if __name__ == "__main__":
    app()


Review comment (suggestion):
It's OK to write empty files...

```suggestion
```

Code after:

    model = fallback_model(model)
    ai = AI(
        model=model,
        temperature=temperature,
    )

    input_path = Path(project_path).absolute()
    memory_path = input_path / f"{run_prefix}memory"
    workspace_path = input_path / f"{run_prefix}workspace"
    archive_path = input_path / f"{run_prefix}archive"

    initial_run = not os.path.exists(memory_path) and not os.path.exists(workspace_path)
    dbs = DBs(
        memory=DB(memory_path),
        logs=DB(memory_path / "logs"),
        input=DB(input_path),
        workspace=DB(workspace_path),
        preprompts=DB(Path(__file__).parent / "preprompts"),
        archive=None if initial_run else DB(archive_path),
    )

    steps = STEPS[steps_config]
    for step in steps:
        messages = step(ai, dbs)
        dbs.logs[step.__name__] = json.dumps(messages)

    collect_learnings(model, temperature, steps, dbs)


if __name__ == "__main__":
    app()
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/main.py (Python)

Code before:
    memory_path = input_path / f"{run_prefix}memory"
    workspace_path = input_path / f"{run_prefix}workspace"

    if delete_existing:
        # Delete files and subdirectories in paths
        shutil.rmtree(memory_path, ignore_errors=True)
        shutil.rmtree(workspace_path, ignore_errors=True)

    model = fallback_model(model)

    ai = AI(
        model=model,
        temperature=temperature,
    )

    dbs = DBs(
        memory=DB(memory_path),
        logs=DB(memory_path / "logs"),
        input=DB(input_path),
        workspace=DB(workspace_path),
        preprompts=DB(Path(__file__).parent / "preprompts"),
    )

    steps = STEPS[steps_config]
    for step in steps:
        messages = step(ai, dbs, step_name=step.__name__)
        dbs.logs[step.__name__] = json.dumps(messages)

    collect_learnings(model, temperature, steps, dbs)

    dbs.logs["token_usage"] = ai.format_token_usage_log()

if __name__ == "__main__":
    app()


Review comment (suggestion):
```suggestion
        messages = step(ai, dbs)
```

Code after:

    dbs = DBs(
        memory=DB(memory_path),
        logs=DB(memory_path / "logs"),
        input=DB(input_path),
        workspace=DB(workspace_path),
        preprompts=DB(Path(__file__).parent / "preprompts"),
        archive=DB(archive_path),
    )

    if steps_config not in [
        StepsConfig.EXECUTE_ONLY,
        StepsConfig.USE_FEEDBACK,
        StepsConfig.EVALUATE,
    ]:
        archive(dbs)

    steps = STEPS[steps_config]
    for step in steps:
        messages = step(ai, dbs)
        dbs.logs[step.__name__] = json.dumps(messages)

    if collect_consent():
        collect_learnings(model, temperature, steps, dbs)

    dbs.logs["token_usage"] = ai.format_token_usage_log()


if __name__ == "__main__":
    app()
Repository: AntonOsika/gpt-engineer
File: scripts/benchmark.py (Python)

Code before:
        current_date = datetime.now().strftime("%Y-%m-%d")
        insert_markdown_section(results_path, current_date, table, 2)


def to_emoji(value: bool) -> str:
    return "\U00002705" if value else "\U0000274C"


def insert_markdown_section(file_path, section_title, section_text, level):
    with open(file_path, "r") as file:
        lines = file.readlines()

    header_prefix = "#" * level
    new_section = f"{header_prefix} {section_title}\n\n{section_text}\n\n"

    # Find the first section with the specified level
    line_number = -1
    for i, line in enumerate(lines):
        if line.startswith(header_prefix):
            line_number = i
            break

    if line_number != -1:
        lines.insert(line_number, new_section)
    else:
        print(f"No section of level {level} found.")
        return

    # Write the file
    with open(file_path, "w") as file:
        file.writelines(lines)


def ask_yes_no(question: str) -> bool:
    while True:
        response = input(question + " (y/n): ").lower().strip()
        if response == "y":
            return True
        elif response == "n":
            return False
        else:
            print("Please enter either 'y' or 'n'.")


if __name__ == "__main__":
    run(main)


Review comment (suggestion):
```suggestion
        print(f"Markdown file was of unexpected format. No section of level {level} found. Did not write results.")
```

Code after:
        current_date = datetime.now().strftime("%Y-%m-%d")
        insert_markdown_section(results_path, current_date, table, 2)


def to_emoji(value: bool) -> str:
    return "\U00002705" if value else "\U0000274C"


def insert_markdown_section(file_path, section_title, section_text, level):
    with open(file_path, "r") as file:
        lines = file.readlines()

    header_prefix = "#" * level
    new_section = f"{header_prefix} {section_title}\n\n{section_text}\n\n"

    # Find the first section with the specified level
    line_number = -1
    for i, line in enumerate(lines):
        if line.startswith(header_prefix):
            line_number = i
            break

    if line_number != -1:
        lines.insert(line_number, new_section)
    else:
        print(f"Markdown file was of unexpected format. No section of level {level} found. Did not write results.")
        return

    # Write the file
    with open(file_path, "w") as file:
        file.writelines(lines)


def ask_yes_no(question: str) -> bool:
    while True:
        response = input(question + " (y/n): ").lower().strip()
        if response == "y":
            return True
        elif response == "n":
            return False
        else:
            print("Please enter either 'y' or 'n'.")


if __name__ == "__main__":
    run(main)
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/chat_to_files.py (Python)

Code before:

        # Get the code
        code = match.group(2)

        # Add the file to the list
        files.append((path, code))

    # Get all the text before the first ``` block
    readme = chat.split("```")[0]
    files.append(("README.md", readme))

    # Return the files
    return files


def to_files(chat, workspace):
    workspace["all_output.txt"] = chat

    files = parse_chat(chat)
    for file_name, file_content in files:
        workspace[file_name] = file_content


# Get code content from a code list
def getCodeStrings(input):
    filesPaths = input["file_list.txt"].split("\n")
    filesDict = {}
    for filePath in filesPaths:
        with open(filePath, "r") as file:
            fileData = file.read()
            fileName = os.path.basename(filePath).split("/")[-1]
            filesDict[fileName] = fileData
    return filesDict


# Format file for inputing to chat prompt
def formatFileToInput(fileName, fileContent):
    filestr = f"""
    {fileName}
    ```
    {fileContent}
    ```
    """
    return filestr


Review comment (suggestion):
This looks too cumbersome imo.

We should be able to generate this list of filepaths from the project directory to make it nice for the user!

Could, for example, print the content of the project directory, and ask the user 1. which folders to include and 2. which file extensions to include. And then this is stored to memory.

Code after:

        # Get the code
        code = match.group(2)

        # Add the file to the list
        files.append((path, code))

    # Get all the text before the first ``` block
    readme = chat.split("```")[0]
    files.append(("README.md", readme))

    # Return the files
    return files


def to_files(chat, workspace):
    workspace["all_output.txt"] = chat

    files = parse_chat(chat)
    for file_name, file_content in files:
        workspace[file_name] = file_content


def overwrite_files(chat, dbs, replace_files):
    """
    Replace the AI files to the older local files.
    """
    dbs.workspace["all_output.txt"] = chat

    files = parse_chat(chat)
    for file_name, file_content in files:
        # Verify if the file created by the AI agent was in the input list
        if file_name in replace_files:
            # If the AI created a file from our input list, we replace it.
            with open(replace_files[file_name], "w") as text_file:
                text_file.write(file_content)
        else:
            # If the AI create a new file I don't know where to put it yet
            # maybe we can think in a smarter solution for this in the future
            # like asking the AI where to put it.
            #
            # by now, just add this to the workspace inside .gpteng folder
            print(
                f"Could not find file path for '{file_name}', creating file in workspace"
            )
            dbs.workspace[file_name] = file_content


def get_code_strings(input) -> dict[str, str]:
    """
    Read file_list.txt and return file names and its content.
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/steps.py (Python)

Code before:
def human_review(ai: AI, dbs: DBs):
    review = human_input()
    dbs.memory["review"] = review.to_json()  # type: ignore
    return []


class Config(str, Enum):
    DEFAULT = "default"
    BENCHMARK = "benchmark"
    SIMPLE = "simple"
    TDD = "tdd"
    TDD_PLUS = "tdd+"
    CLARIFY = "clarify"
    RESPEC = "respec"
    EXECUTE_ONLY = "execute_only"
    EVALUATE = "evaluate"
    USE_FEEDBACK = "use_feedback"


# Different configs of what steps to run
STEPS = {
    Config.DEFAULT: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,
        execute_entrypoint,
    ],
    Config.BENCHMARK: [simple_gen, gen_entrypoint],
    Config.SIMPLE: [simple_gen, gen_entrypoint, execute_entrypoint],
    Config.TDD: [
        gen_spec,
        gen_unit_tests,
        gen_code,
        gen_entrypoint,
        execute_entrypoint,
        human_review,
    ],
    Config.TDD_PLUS: [
        gen_spec,
        gen_unit_tests,
        gen_code,
        fix_code,
        gen_entrypoint,
        execute_entrypoint,
        human_review,
    ],
    Config.CLARIFY: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,
        execute_entrypoint,

Review comment (suggestion):
```suggestion
        execute_entrypoint,
        human_review,
```

Code after:
def human_review(ai: AI, dbs: DBs):
    review = human_input()
    dbs.memory["review"] = review.to_json()  # type: ignore
    return []


class Config(str, Enum):
    DEFAULT = "default"
    BENCHMARK = "benchmark"
    SIMPLE = "simple"
    TDD = "tdd"
    TDD_PLUS = "tdd+"
    CLARIFY = "clarify"
    RESPEC = "respec"
    EXECUTE_ONLY = "execute_only"
    EVALUATE = "evaluate"
    USE_FEEDBACK = "use_feedback"


# Different configs of what steps to run
STEPS = {
    Config.DEFAULT: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,
        execute_entrypoint,
        human_review,
    ],
    Config.BENCHMARK: [simple_gen, gen_entrypoint],
    Config.SIMPLE: [simple_gen, gen_entrypoint, execute_entrypoint],
    Config.TDD: [
        gen_spec,
        gen_unit_tests,
        gen_code,
        gen_entrypoint,
        execute_entrypoint,
        human_review,
    ],
    Config.TDD_PLUS: [
        gen_spec,
        gen_unit_tests,
        gen_code,
        fix_code,
        gen_entrypoint,
        execute_entrypoint,
        human_review,
    ],
    Config.CLARIFY: [
        clarify,
        gen_clarified_code,
        gen_entrypoint,
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/core/ai.py (Python)

Code before:
from __future__ import annotations

import json
import logging
import os

from typing import List, Optional, Union

import backoff
import openai

from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.chat_models.base import BaseChatModel
from langchain.schema import (
    AIMessage,
    HumanMessage,
    SystemMessage,
    messages_from_dict,
    messages_to_dict,
)
from langchain_community.chat_models import AzureChatOpenAI
from langchain_openai import ChatOpenAI

from gpt_engineer.core.token_usage import TokenUsageLog

# Type hint for a chat message
Message = Union[AIMessage, HumanMessage, SystemMessage]

# Set up logging
logger = logging.getLogger(__name__)


class AI:
    def __init__(
        self,
        model_name="gpt-4-1106-preview",
        temperature=0.1,
        azure_endpoint="",
        streaming=True,
    ):
        """
        Initialize the AI class.

        Parameters
        ----------
        model_name : str, optional

Review comment (suggestion):
When looking up [docs for langchain_community.chat_models.AzureChatOpenAI](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.azure_openai.AzureChatOpenAI.html#) they state that it is deprecated.

The [langchain-openai README](https://pypi.org/project/langchain-openai/) state that you should use:

```
from langchain_openai import AzureChatOpenAI
```

Code after:
from __future__ import annotations

import json
import logging
import os

from typing import List, Optional, Union

import backoff
import openai

from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.chat_models.base import BaseChatModel
from langchain.schema import (
    AIMessage,
    HumanMessage,
    SystemMessage,
    messages_from_dict,
    messages_to_dict,
)
from langchain_openai import AzureChatOpenAI, ChatOpenAI

from gpt_engineer.core.token_usage import TokenUsageLog

# Type hint for a chat message
Message = Union[AIMessage, HumanMessage, SystemMessage]

# Set up logging
logger = logging.getLogger(__name__)


class AI:
    def __init__(
        self,
        model_name="gpt-4-1106-preview",
        temperature=0.1,
        azure_endpoint="",
        streaming=True,
    ):
        """
        Initialize the AI class.

        Parameters
        ----------
        model_name : str, optional
            The name of the model to use, by default "gpt-4".
Repository: AntonOsika/gpt-engineer
File: gpt_engineer/core/project_config.py (Python)

Code before:
"""
Functions for reading and writing the `gpt-engineer.toml` configuration file.

The `gpt-engineer.toml` file is a TOML file that contains project-specific configuration used by the GPT Engineer CLI and gptengineer.app.
"""

from dataclasses import asdict, dataclass, field
from pathlib import Path

import tomlkit

default_config_filename = "gpt-engineer.toml"

example_config = """
[project]
base_dir = "./frontend"  # base directory to operate in (for monorepos)
src_dir = "./src"        # source directory (under the base directory) from which context will be retrieved

[run]
build = "npm run build"
test = "npm run test"
lint = "quick-lint-js"

[gptengineer-app]  # this namespace is used for gptengineer.app, may be used for internal experiments
project_id = "..."

# we support multiple OpenAPI schemas, used as context for the LLM
openapi = [
    { url = "https://api.gptengineer.app/openapi.json" },
    { url = "https://some-color-translating-api/openapi.json" },
]
"""


@dataclass
class _ProjectConfig:
    base_dir: str | None = None
    src_dir: str | None = None


@dataclass
class _RunConfig:

Review comment (suggestion):
I'm not sure about this section. Would perhaps make more sense with a `paths` section, such that it can be just:

```suggestion
[paths]
base = "./frontend"  # base directory to operate in (for monorepos)
src = "./src"        # source directory (under the base directory) from which context will be retrieved
```

Code after:
"""
Functions for reading and writing the `gpt-engineer.toml` configuration file.

The `gpt-engineer.toml` file is a TOML file that contains project-specific configuration used by the GPT Engineer CLI and gptengineer.app.
"""

from dataclasses import asdict, dataclass, field
from pathlib import Path

import tomlkit

default_config_filename = "gpt-engineer.toml"

example_config = """
[paths]
base = "./frontend"  # base directory to operate in (for monorepos)
src = "./src"        # source directory (under the base directory) from which context will be retrieved

[run]
build = "npm run build"
test = "npm run test"
lint = "quick-lint-js"

[gptengineer-app]  # this namespace is used for gptengineer.app, may be used for internal experiments
project_id = "..."

# we support multiple OpenAPI schemas, used as context for the LLM
openapi = [
    { url = "https://api.gptengineer.app/openapi.json" },
    { url = "https://some-color-translating-api/openapi.json" },
]
"""


@dataclass
class _ProjectConfig:
    base_dir: str | None = None
    src_dir: str | None = None


@dataclass
class _RunConfig: