p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/alfred/berlin/coding_agent/next_action_eval/handcrafted/generate_eval_qwen_0.6_base.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:20:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --mem=200GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/alfred/logs/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/alfred/logs/%x_%j.log
#SBATCH --job-name=qwen3_coder_0.6b_base_handcrafted


module load CUDA/12.8
source .venv/bin/activate


model_path="Qwen/Qwen3-0.6b"
input_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"

experiment_name="qwen3_coder_0.6b_base_handcrafted"
wandb_tags="next_action_eval handcrafted qwen3_30b"


# Generation and evaluation config
limit=-1
accept_threshold=0.7
context_length=40_960
problem_length=20_000
wandb_name="${experiment_name}_${SLURM_JOB_ID}"
judge_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"
output_file="data/eval/results/next_action_eval_handcrafted/${wandb_name}_generations.jsonl"
eval_input_file="$output_file"
eval_output_file="data/eval/results/next_action_eval_handcrafted/${wandb_name}_evaluation.json"
wandb_eval_type="next_action_handcrafted"

echo "================================================"
echo "Generating generations ..."
echo "================================================"


python src/evaluation/sglang_generations.py \
  --model_path $model_path \
  --context_length $context_length \
  --problem_length $problem_length \
  --input_file $input_file \
  --output_file $output_file \
  --limit $limit \


echo "================================================"
echo "Evaluating generations ..."
echo "================================================"

python src/evaluation/sglang_eval.py \
  --model_path $judge_model_path \
  --context_length $context_length \
  --problem_length $problem_length \
  --input_file $eval_input_file \
  --output_file $eval_output_file \
  --accept_threshold $accept_threshold \
  --wandb_name ${wandb_name} \
  --wandb_tags ${wandb_tags} \
  --wandb_eval_type ${wandb_eval_type} \
  --limit $limit \