p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] data/eval/hello_world_insert_evaluations.jsonl (jsonl)
{
  "metadata": {
    "model_path": "Qwen/Qwen3-Coder-30B-A3B-Instruct",
    "input_file": "data/eval/handcrafted_test_cases/handcrafted_unit_tests.jsonl",
    "output_file": "data/eval/hello_world_insert_evaluations.jsonl",
    "limit": -1,
    "system_prompt_file": "data/prompts/system_prompt_eval_judger.md",
    "prompt_file": "data/prompts/command_evaluation_prompt.txt",
    "judge_name": "default",
    "context_length": 40960,
    "problem_length": 40960,
    "accept_threshold": 0.7,
    "max_attempts": 6,
    "timeout": 30.0,
    "concurrency": 64,
    "max_connections": 256,
    "keepalive": 60
  },
  "judge_eval_scores": {
    "total_test_cases": 6,
    "num_correct": 0,
    "pass_rate": 0.0,
    "avg_score": 0.0,
    "accept_threshold": 0.7
  },
  "generation_results": [
    {
      "task_id": "eval_cursor_movement/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    },
    {
      "task_id": "eval_delete/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    },
    {
      "task_id": "eval_file_switch/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    },
    {
      "task_id": "eval_insert/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    },
    {
      "task_id": "eval_replace/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    },
    {
      "task_id": "eval_terminal_invocation/0",
      "error": "'expected_command'",
      "is_correct": 0,
      "average_score": 0.0
    }
  ]
}