[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --job-name=eval_miles_run


module load CUDA/12.8
source .venv/bin/activate

# TODO set these
RUN_ID="39103"
RUN_NAME="eval_qwen1.7B_no_lora_bs_32_39103"
BASE_MODEL=Qwen3-1.7B

CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/${RUN_ID}/checkpoints"
HF_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface"

USE_LORA=false
# Generation
generator_host="0.0.0.0"
generator_port=30000
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/unit_test_cases.jsonl"
output_dir="data/eval/output/${RUN_ID}"

mkdir -p $output_dir

# Evaluation
judge_host="localhost"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name=$RUN_NAME
wandb_project="crowd-pilot-miles"
wandb_eval_type="unit-test-v1"



# Check if this is a lora run or full fine tune
get_latest_iter() {
    ls -1 "$CKPT_DIR" 2>/dev/null | grep -E '^iter_[0-9]+$' | sort -t_ -k2 -n | tail -1
}

latest_iter=$(get_latest_iter)
if [[ -f "$CKPT_DIR/$latest_iter/adapter_model.safetensors" ]]; then
    echo "LoRA adapters found. Evaluating accordingly."
    USE_LORA=true
else
    USE_LORA=false
    echo "No LoRA adapters found. Evaluating as full fine-tune."
fi


step=0
generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
# evaluate base model for "step 0"
# python src/evaluation/sglang_generations.py \
    # --test_cases_file $test_cases_file \
    # --generations_file $generations_file \
    # --system_prompt_file "data/prompts/swe_mini_v1.md" \
    # --model_path $HF_DIR/$BASE_MODEL \
    # --context_length $context_length \
    # --server_host $generator_host \
    # --server_port $generator_port \
    # --temperature $temperature \
    # --top_p $top_p \
    # --top_k $top_k \
    # --min_p $min_p \
    # --num_samples $num_samples \
    # --mem_fraction_static 0.80 \
    # --tp_size 4 \

# python src/evaluation/sglang_eval.py \
    # --wandb_name $wandb_name \
    # --wandb_id run_$RUN_ID \
    # --generations_file $generations_file \
    # --evaluations_file $evaluations_file \
    # --judge_model_path $judge_model_path \
    # --server_host $judge_host \
    # --server_port $judge_port \
    # --context_length $judge_context_length \
    # --judge_prompt_file $judge_prompt_file \
    # --temperature $temperature \
    # --wandb_eval_type $wandb_eval_type \
    # --eval_step $step 


for dir in $(ls -d "${CKPT_DIR}"/iter_*/ | sort -V); do
    step_name=$(basename "$dir")
    step=$((10#${step_name#iter_}))
    generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
    evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
    if $USE_LORA ; then
        echo "LoRA adapter dir: $dir, Step: $step"
        GENERATION_ARGS=(
            --model_path $HF_DIR/$BASE_MODEL
            --lora_paths $dir
        )
    else
        cp $HF_DIR/$BASE_MODEL/*.json $dir
        GENERATION_ARGS=(
            --model_path $dir
        )
        echo 'Not using LORA'
    fi

    # Generate
    echo "generating.."
    python src/evaluation/sglang_generations.py \
        --test_cases_file $test_cases_file \
        --generations_file $generations_file \
        --system_prompt_file "data/prompts/swe_mini_v1.md" \
        --context_length $context_length \
        --server_host $generator_host \
        --server_port $generator_port \
        --temperature $temperature \
        --top_p $top_p \
        --top_k $top_k \
        --min_p $min_p \
        --num_samples $num_samples \
        --mem_fraction_static 0.80 \
        --tp_size 4 \
        ${GENERATION_ARGS[@]}

    # Evaluate
    echo "evaluating.."

    python src/evaluation/sglang_eval.py \
    --wandb_name $wandb_name \
    --wandb_id run_$RUN_ID \
    --generations_file $generations_file \
    --evaluations_file $evaluations_file \
    --judge_model_path $judge_model_path \
    --server_host $judge_host \
    --server_port $judge_port \
    --context_length $judge_context_length \
    --judge_prompt_file $judge_prompt_file \
    --temperature $temperature \
    --wandb_eval_type $wandb_eval_type \
    --eval_step $step
done