p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] tab-model-eval/data/eval/local_logs/eval_handcrafted_13032800/metadata.json (json)
{
  "run_id": "eval_handcrafted_13032800",
  "run_name": "qwen1.7B_lora_bs_16_13032800",
  "project": "llm-coding-agent",
  "config": {
    "batch_mode": true,
    "num_jobs": 11
  },
  "tags": [
    "val_mini",
    "judge_eval"
  ],
  "created_at": "2026-01-06T17:59:38"
}
[tab] tab-model-eval/data/eval/local_logs/handcrafted/eval_handcrafted_13032806/metadata.json (json)
{
  "run_id": "eval_handcrafted_13032806",
  "run_name": "qwen32B_no_lora_bs_32_13032806",
  "project": "llm-coding-agent",
  "config": {
    "batch_mode": true,
    "num_jobs": 6
  },
  "tags": [
    "val_mini",
    "judge_eval"
  ],
  "created_at": "2026-01-06T22:32:13"
}
[tab] tab-model-eval/data/eval/local_logs/handcrafted/eval_handcrafted_13032806/metrics.jsonl (jsonl)
{"timestamp": "2026-01-06T22:32:48", "eval_step": 0, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.2619047619047619, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.2619047619047619, "unit-test-v1/total_exact_match_pass_at_n": 6}
{"timestamp": "2026-01-06T22:33:20", "eval_step": 1000, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.2761904761904762, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.2761904761904762, "unit-test-v1/total_exact_match_pass_at_n": 6}
{"timestamp": "2026-01-06T22:33:52", "eval_step": 2000, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.27142857142857146, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.27142857142857146, "unit-test-v1/total_exact_match_pass_at_n": 6}
{"timestamp": "2026-01-06T22:34:23", "eval_step": 3000, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.27142857142857146, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.27142857142857146, "unit-test-v1/total_exact_match_pass_at_n": 6}
{"timestamp": "2026-01-06T22:34:56", "eval_step": 4000, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.26666666666666666, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.26666666666666666, "unit-test-v1/total_exact_match_pass_at_n": 6}
{"timestamp": "2026-01-06T22:35:27", "eval_step": 5000, "unit-test-v1/total_test_cases": 21, "unit-test-v1/num_samples_per_task": 10, "unit-test-v1/total_judge_avg_at_n": 0.26666666666666666, "unit-test-v1/total_judge_pass_at_n": 6, "unit-test-v1/total_exact_match_avg_at_n": 0.26666666666666666, "unit-test-v1/total_exact_match_pass_at_n": 6}
[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_8b_lora_bs_16_13066057.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

# Run the training script
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
============================================
SBATCH Environment:
  MODEL_SIZE: 8b
  BATCH_SIZE: 16
  USE_LORA:   true
  CP_SIZE:    1
  SLURM_JOB_ID: 13066057
============================================

  This stage is deprecated. Please consider moving to a new stage (2026
or newer)


Due to MODULEPATH changes, the following have been reloaded:
  1) make/4.4.1

The following have been reloaded with a version change:
  1) GCCcore/.14.3.0 => GCCcore/.13.3.0     4) binutils/2.44 => binutils/.2.42
  2) Stages/2026 => Stages/2025             5) zlib/1.3.1 => zlib/.1.3.1
  3) StdEnv/2026 => StdEnv/2025

srun: error: jwb0732: task 3: Terminated
srun: error: jwb0730: task 1: Terminated
srun: error: jwb0731: task 2: Terminated
srun: error: jwb0729: task 0: Exited with exit code 1
srun: Force Terminated StepId=13066057.0
[tab] slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/sbatch_template.sh (shellscript)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load Stages/2025
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

# Run the training script
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
[tab] tab-model-eval/data/eval/local_logs/handcrafted/eval_handcrafted_13032812/metadata.json (json)
{
  "run_id": "eval_handcrafted_13032812",
  "run_name": "qwen4B_no_lora_bs_16_13032812",
  "project": "llm-coding-agent",
  "config": {
    "batch_mode": true,
    "num_jobs": 11
  },
  "tags": [
    "val_mini",
    "judge_eval"
  ],
  "created_at": "2026-01-06T22:33:46"
}