p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] src/serving/launch_sglang.sh (shellscript)
module load CUDA/12.8

source .venv/bin/activate

model_id="Qwen/Qwen3-0.6b"
# model_id="Qwen/Qwen3-Coder-30B-A3B-Instruct"
# model_id="Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[tab] src/evaluation/evaluate_client.py (python)
import openai
import json
import os
import tyro
from dataclasses import dataclass
from tqdm import tqdm


@dataclass
class Args:
    name: str = "hello_world_insert"
    input_path: str = "data/incremental_test_cases/"
    output_path: str = "data/evaluation_results/"
    system_prompt_file: str = "data/system_prompts/minimal_v1.md"
    api_base_url: str = "http://hai005:30000/v1"


def load_dataset(filepath):
    """Yields each test case from the jsonl file."""
    with open(filepath, "r") as f:
        for line in f:
            yield json.loads(line)


def format_prompt_for_model(context, system_prompt):
    """Formats the context into a single string for the LLM."""
    prompt_str = f"SYSTEM: {system_prompt}
"
    for message in context:
        role = message["role"].upper()
        content = message["content"]
        prompt_str += f"{role}: {content}
"
    prompt_str += "ASSISTANT:"
    return prompt_str


def run_evaluation_client(args: Args):
    client = openai.OpenAI(base_url=args.api_base_url, api_key="EMPTY")

    test_cases = list(load_dataset(args.input_path + args.name + ".jsonl"))
    system_prompt = open(args.system_prompt_file, "r").read()

    num_correct = 0
    results = []

    print(f"Running inference on {len(test_cases)} test cases...")
    for test_case in tqdm(test_cases):
        formatted_messages = [{"role": "system", "content": system_prompt}]

        for turn in test_case["context"]:
            formatted_messages.append(turn)

        try:
            response = client.chat.completions.create(
                model="default",
                messages=formatted_messages,
                max_tokens=256,
                temperature=0.0,
                stop=["
```", "ASSISTANT:", "USER:"],
            )
            generated_text = response.choices[0].message.content
        except Exception as e:
            print(f"API Error on task {test_case['task_id']}: {e}")
            generated_text = ""  # Mark as incorrect on error

        generated_command = generated_text
        expected_command = test_case["expected_final_response"]
        is_correct = 1 if generated_command == expected_command else 0
        num_correct += is_correct
        results.append(
            {
                "task_id": test_case["task_id"],
                "generated": generated_command,
                "expected": expected_command,
                "score": is_correct,
            }
        )

    os.makedirs(args.output_path, exist_ok=True)
    json.dump(results, open(args.output_path + args.name + ".json", "w"))
    # 6. Report final score
    pass_rate = (num_correct / len(test_cases)) * 100
    print("
--- Evaluation Complete ---")
    print(f"Total Test Cases: {len(test_cases)}")
    print(f"Correct: {num_correct}")
    print(f"Pass Rate (Exact Match): {pass_rate:.2f}%")


if __name__ == "__main__":
    args = tyro.cli(Args)
    run_evaluation_client(args)
[tab] src/input_pipeline/trace_to_testcases.py (python)
import json
import re
import tyro
from dataclasses import dataclass


@dataclass
class Args:
    name: str = "hello_world_insert"
    input_path: str = "data/handcrafted_traces/"
    output_path: str = "data/incremental_test_cases/"


def parse_md_file(file_path):
    with open(file_path, "r") as file:
        content = file.read()

    sections = re.split(r"# (User|Assistant)", content)
    sections = [s.strip() for s in sections if s.strip()]

    parsed_data = []
    current_dialogue = None

    for index, section in enumerate(sections):
        if section.startswith("User") or section.startswith("Assistant"):
            if current_dialogue is not None:
                parsed_data.append(current_dialogue)
            current_dialogue = {
                "role": section.lower(),
                "content": sections[index + 1].strip().replace("
```", "```"),
            }

    if current_dialogue is not None:
        parsed_data.append(current_dialogue)

    return parsed_data


def create_incremental_test_cases(parsed_data, task_id):
    test_cases = []

    task_number = 0
    for i in range(1, len(parsed_data) - 1, 2):  # Ensure context ends with user content
        context = parsed_data[: i + 1]
        expected_response = (
            parsed_data[i + 1]["content"]
            if parsed_data[i + 1]["role"] == "assistant"
            else ""
        )

        test_case = {
            "task_id": f"{task_id}/{task_number}",
            "context": context,
            "expected_final_response": expected_response,
        }
        test_cases.append(test_case)
        task_number += 1

    return test_cases


def write_jsonl(test_cases, output_file):
    with open(output_file, "w") as file:
        for test_case in test_cases:
            file.write(json.dumps(test_case) + "
")


# Main execution
if __name__ == "__main__":
    args = tyro.cli(Args)

    task_id = args.name
    input_file_path = args.input_path + args.name + ".md"
    output_file_path = args.output_path + args.name + ".jsonl"

    parsed_data = parse_md_file(input_file_path)
    incremental_test_cases = create_incremental_test_cases(parsed_data, task_id)
    write_jsonl(incremental_test_cases, output_file_path)
[tab] data/system_prompts/simple_v1.md (markdown)
You are a bash assistant.
- Read the dialogue and continue the task.
- Output ONE final answer: a single bash code block with the exact command(s) to run.
- Do not include explanations or extra text.
- Do not echo prior output.
- If multiple commands are required, chain them with &&.
- Never include a leading shell prompt ($).
- Use POSIX-compatible tools unless the context shows otherwise.
[content] data/system_prompts/crowd_code_v1.md (markdown)
You are a helpful assistant that can interact multiple times with a computer shell to solve programming tasks.
Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||).

Format your response as shown in <format_example>.

<format_example>
```bash
your_command_here
```
</format_example>

Failure to follow these rules will cause your response to be rejected.
[tab] src/input_pipeline/trace_to_testcases.py (python)
import json
import os
import re
import tyro
from dataclasses import dataclass


@dataclass
class Args:
    name: str = "hello_world_insert"
    input_path: str = "data/handcrafted_traces/"
    output_path: str = "data/incremental_test_cases/"


def parse_md_file(file_path):
    with open(file_path, "r") as file:
        content = file.read()

    sections = re.split(r"# (User|Assistant)", content)
    sections = [s.strip() for s in sections if s.strip()]

    parsed_data = []
    current_dialogue = None

    for index, section in enumerate(sections):
        if section.startswith("User") or section.startswith("Assistant"):
            if current_dialogue is not None:
                parsed_data.append(current_dialogue)
            current_dialogue = {
                "role": section.lower(),
                "content": sections[index + 1].strip().replace("
```", "```"),
            }

    if current_dialogue is not None:
        parsed_data.append(current_dialogue)

    return parsed_data


def create_incremental_test_cases(parsed_data, task_id):
    test_cases = []

    task_number = 0
    for i in range(1, len(parsed_data) - 1, 2):  # Ensure context ends with user content
        context = parsed_data[: i + 1]
        expected_response = (
            parsed_data[i + 1]["content"]
            if parsed_data[i + 1]["role"] == "assistant"
            else ""
        )

        test_case = {
            "task_id": f"{task_id}/{task_number}",
            "context": context,
            "expected_final_response": expected_response,
        }
        test_cases.append(test_case)
        task_number += 1

    return test_cases


def write_jsonl(test_cases, output_file):
    with open(output_file, "w") as file:
        for test_case in test_cases:
            file.write(json.dumps(test_case) + "
")


# Main execution
if __name__ == "__main__":
    args = tyro.cli(Args)

    task_id = args.name
    input_file_path = args.input_path + args.name + ".md"
    output_file_path = args.output_path + args.name + ".jsonl"
    os.makedirs(args.output_path, exist_ok=True)

    parsed_data = parse_md_file(input_file_path)
    incremental_test_cases = create_incremental_test_cases(parsed_data, task_id)
    write_jsonl(incremental_test_cases, output_file_path)
[tab] data/handcrafted_traces/hello_world.md (markdown)
# Assistant
``` bash
ls -la
```

# User
<stdout>
[alfred.nguyen@hai-login1.haicore.berlin:~] $ ls -la
total 336
drwxrwx--- 3 alfred.nguyen alfred.nguyen   318 Nov 13 18:05 .
drwxr-xr-x 9 alfred.nguyen alfred.nguyen   171 Nov 12 15:29 ..
[alfred.nguyen@hai-login1.haicore.berlin:~] $ 
</stdout>

# Assistant
``` bash
cat <<'EOF' > hello_world.py
print("Hello, World!")
EOF && cat -n hello_world.py
```

# User
<stdout>
     1  print("Hello, World!")
</stdout>

# Assistant
```bash
python hello_world.py
```

# User
<stdout>
Hello, World!
</stdout>

# Assistant
```bash
echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT
```