[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --job-name=eval_miles_run


module load CUDA/12.8
source .venv/bin/activate

# TODO set these
RUN_ID="38875"
RUN_NAME="eval_qwen1.7B_lora_bs_32_38875"
BASE_MODEL=Qwen3-1.7B
CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/${RUN_ID}/checkpoints"
HF_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface"

USE_LORA=false
# Generation
generator_host="0.0.0.0"
generator_port=30000
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/unit_test_cases.jsonl"
output_dir="data/eval/output/${RUN_ID}"

mkdir -p $output_dir

# Evaluation
judge_host="hai008"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name=$RUN_NAME
wandb_project="crowd-pilot-miles"
wandb_eval_type="unit-test-v1"



# Check if this is a lora run or full fine tune
get_latest_iter() {
    ls -1 "$CKPT_DIR" 2>/dev/null | grep -E '^iter_[0-9]+$' | sort -t_ -k2 -n | tail -1
}

latest_iter=$(get_latest_iter)
if [[ -f "$CKPT_DIR/$latest_iter/adapter_model.safetensors" ]]; then
    echo "LoRA adapters found. Evaluating accordingly."
    USE_LORA=true
else
    USE_LORA=false
    echo "No LoRA adapters found. Evaluating as full fine-tune."
fi


step=0
generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
# evaluate base model for "step 0"
python src/evaluation/sglang_generations.py \
    --test_cases_file $test_cases_file \
    --generations_file $generations_file \
    --system_prompt_file "data/prompts/swe_mini_v1.md" \
    --model_path $HF_DIR/$BASE_MODEL \
    --context_length $context_length \
    --server_host $generator_host \
    --server_port $generator_port \
    --temperature $temperature \
    --top_p $top_p \
    --top_k $top_k \
    --min_p $min_p \
    --num_samples $num_samples \
    --mem_fraction_static 0.80 \
    --tp_size 2 \

python src/evaluation/sglang_eval.py \
    --wandb_name $wandb_name \
    --wandb_id run_$RUN_ID \
    --generations_file $generations_file \
    --evaluations_file $evaluations_file \
    --judge_model_path $judge_model_path \
    --server_host $judge_host \
    --server_port $judge_port \
    --context_length $judge_context_length \
    --judge_prompt_file $judge_prompt_file \
    --temperature $temperature \
    --wandb_eval_type $wandb_eval_type \
    --eval_step $step \
    --no_launch_server


# for dir in $(ls -d "${CKPT_DIR}"/iter_*/ | sort -V); do
    # step_name=$(basename "$dir")
    # step=$((10#${step_name#iter_}))
    # generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
    # evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
    # if $USE_LORA ; then
        # echo "LoRA adapter dir: $dir, Step: $step"
        # GENERATION_ARGS=(
            # --model_path $HF_DIR/$BASE_MODEL
            # --lora_paths $dir
        # )
    # else
        # GENERATION_ARGS=(
            # --model_path $dir
        # )
        # echo 'Not using LORA'
    # fi

    # # Generate
    # echo "generating.."
    # python src/evaluation/sglang_generations.py \
        # --test_cases_file $test_cases_file \
        # --generations_file $generations_file \
        # --system_prompt_file "data/prompts/swe_mini_v1.md" \
        # --context_length $context_length \
        # --server_host $generator_host \
        # --server_port $generator_port \
        # --temperature $temperature \
        # --top_p $top_p \
        # --top_k $top_k \
        # --min_p $min_p \
        # --num_samples $num_samples \
        # --mem_fraction_static 0.80 \
        # --tp_size 2 \
        # ${GENERATION_ARGS[@]}

    # # Evaluate
    # echo "evaluating.."

    # python src/evaluation/sglang_eval.py \
    # --wandb_name $wandb_name \
    # --wandb_id run_$RUN_ID \
    # --generations_file $generations_file \
    # --evaluations_file $evaluations_file \
    # --judge_model_path $judge_model_path \
    # --server_host $judge_host \
    # --server_port $judge_port \
    # --context_length $judge_context_length \
    # --judge_prompt_file $judge_prompt_file \
    # --temperature $temperature \
    # --wandb_eval_type $wandb_eval_type \
    # --eval_step $step \
    # --no_launch_server
# done