[tab] slurm/jobs/alfred/berlin/coding_agent/next_action_eval/handcrafted/generate_eval_qwen_0.6b.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --mem=200GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/alfred/logs/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/alfred/logs/%x_%j.log
#SBATCH --job-name=qwen_0.6b_handcrafted


module load CUDA/12.8
source .venv/bin/activate


# Experiment name and wandb config
model_path="qwen3_600m_full_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-17-40"
input_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"

experiment_name=${model_path%%/*}_handcrafted
wandb_tags="next_action_eval handcrafted qwen3_0.6b"

workspace="/fast/project/HFMI_SynergyUnit/tab_model/data/outputs"
workspace_hf="${workspace}/hf"

# Generation and evaluation config
limit=-1
accept_threshold=0.7
context_length=40_960
problem_length=20_000
server_port=$((30000 + (SLURM_JOB_ID % 10000)))
wandb_name="${experiment_name}_${SLURM_JOB_ID}"
judge_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"
output_file="data/eval/results/next_action_eval_handcrafted/${wandb_name}_generations.jsonl"
eval_input_file="$output_file"
eval_output_file="data/eval/results/next_action_eval_handcrafted/${wandb_name}_evaluation.json"
wandb_eval_type="next_action_handcrafted"


echo "================================================"
echo "Finding latest checkpoint ..."
echo "================================================"

latest_ckpt=$(find "${workspace}/${model_path}/checkpoints" -maxdepth 1 -type d -regex ".*\(last\|latest\).*" | sort | tail -n 1)

echo "Latest checkpoint: ${latest_ckpt}"

# exit if latest_ckpt is not found
if [ -z "${latest_ckpt}" ]; then
    echo "Latest checkpoint not found"
    exit 1
fi

nemo_ckpt_path="${latest_ckpt}"
hf_ckpt_path="${workspace_hf}/${model_path}"

# convert if hf model does not exist
echo "Nemo model path: ${nemo_ckpt_path}"
echo "HF model path: ${hf_ckpt_path}"

if [ ! -d "${hf_ckpt_path}" ]; then
    echo "HF model does not exist, converting model to HF format ..."

    nemo llm export --yes \
        path="${nemo_ckpt_path}" \
        target="hf" \
        overwrite=True \
        output_path="${hf_ckpt_path}"
else 
    echo "HF model was found, skipping conversion ..."
fi


echo "================================================"
echo "Generating generations ..."
echo "================================================"

model_path=$hf_ckpt_path

python src/evaluation/sglang_generations.py \
  --model_path $model_path \
  --context_length $context_length \
  --problem_length $problem_length \
  --input_file $input_file \
  --output_file $output_file \
  --limit $limit \
  --server_port $server_port \


echo "================================================"
echo "Evaluating generations ..."
echo "================================================"

python src/evaluation/sglang_eval.py \
  --model_path $judge_model_path \
  --context_length $context_length \
  --problem_length $problem_length \
  --input_file $eval_input_file \
  --output_file $eval_output_file \
  --accept_threshold $accept_threshold \
  --server_port $server_port \
  --wandb_name ${wandb_name} \
  --wandb_tags ${wandb_tags} \
  --wandb_eval_type ${wandb_eval_type} \
  --limit $limit \