[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run_batch.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --job-name=eval_miles_run


module load CUDA/12.8
source .venv/bin/activate

# TODO set these
RUN_NAME=$1

# Extract RUN_ID (last underscore-separated field)
RUN_ID="${RUN_NAME##*_}"

# Extract model size from RUN_NAME and map to Qwen3 model name
# Pattern: qwen{size}_lora_... -> Qwen3-{size}
model_prefix="${RUN_NAME%%_*}"  # e.g., "qwen1.7B"
model_size="${model_prefix#qwen}"  # e.g., "1.7B"

# Map to official Qwen3 model names
case "$model_size" in
    0.6B|0.6b)   BASE_MODEL="Qwen3-0.6B" ;;
    1.7B|1.7b)   BASE_MODEL="Qwen3-1.7B" ;;
    4B|4b)       BASE_MODEL="Qwen3-4B" ;;
    8B|8b)       BASE_MODEL="Qwen3-8B" ;;
    14B|14b)     BASE_MODEL="Qwen3-14B" ;;
    32B|32b)     BASE_MODEL="Qwen3-32B" ;;
    30B-A3B|30b-a3b)     BASE_MODEL="Qwen3-30B-A3B" ;;
    235B-A22B|235b-a22b) BASE_MODEL="Qwen3-235B-A22B" ;;
    *)
        echo "Error: Unknown model size '$model_size' in RUN_NAME '$RUN_NAME'"
        exit 1
        ;;
esac

echo "Extracted from RUN_NAME: RUN_ID=$RUN_ID, BASE_MODEL=$BASE_MODEL"

CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/${RUN_ID}/checkpoints"
HF_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface"

USE_LORA=false
# Generation
generator_host="0.0.0.0"
generator_port=30000
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20

test_cases_file="data/eval/handcrafted_test_cases/unit_test_cases.jsonl"
output_dir="data/eval/output/${RUN_ID}"

mkdir -p $output_dir

# Evaluation
judge_host="localhost"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name=$RUN_NAME
wandb_project="crowd-pilot-miles"
wandb_eval_type="unit-test-v1"



# Check if this is a lora run or full fine tune
get_latest_iter() {
    ls -1 "$CKPT_DIR" 2>/dev/null | grep -E '^iter_[0-9]+$' | sort -t_ -k2 -n | tail -1
}

latest_iter=$(get_latest_iter)
if [[ -f "$CKPT_DIR/$latest_iter/adapter_model.safetensors" ]]; then
    echo "LoRA adapters found. Evaluating accordingly."
    USE_LORA=true
else
    USE_LORA=false
    echo "No LoRA adapters found. Evaluating as full fine-tune."
fi


step=0
generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
# evaluate base model for "step 0"
python src/evaluation/sglang_generations.py \
    --test_cases_file $test_cases_file \
    --generations_file $generations_file \
    --system_prompt_file "data/prompts/swe_mini_v1.md" \
    --model_path $HF_DIR/$BASE_MODEL \
    --context_length $context_length \
    --server_host $generator_host \
    --server_port $generator_port \
    --temperature $temperature \
    --top_p $top_p \
    --top_k $top_k \
    --num_samples $num_samples \
    --mem_fraction_static 0.80 \
    --tp_size 4 \

python src/evaluation/sglang_eval.py \
    --wandb_name $wandb_name \
    --wandb_id my_eval_run_$RUN_ID \
    --generations_file $generations_file \
    --evaluations_file $evaluations_file \
    --judge_model_path $judge_model_path \
    --server_host $judge_host \
    --server_port $judge_port \
    --context_length $judge_context_length \
    --judge_prompt_file $judge_prompt_file \
    --temperature $temperature \
    --wandb_eval_type $wandb_eval_type \
    --eval_step $step 


for dir in $(ls -d "${CKPT_DIR}"/iter_*/ | sort -V); do
    step_name=$(basename "$dir")
    step=$((10#${step_name#iter_}))
    generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
    evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
    if $USE_LORA ; then
        echo "LoRA adapter dir: $dir, Step: $step"
        GENERATION_ARGS=(
            --model_path $HF_DIR/$BASE_MODEL
            --lora_paths $dir
        )
    else
        cp $HF_DIR/$BASE_MODEL/*.json $dir
        GENERATION_ARGS=(
            --model_path $dir
        )
        echo 'Not using LORA'
    fi

    # Generate
    echo "generating.."
    python src/evaluation/sglang_generations.py \
        --test_cases_file $test_cases_file \
        --generations_file $generations_file \
        --system_prompt_file "data/prompts/swe_mini_v1.md" \
        --context_length $context_length \
        --server_host $generator_host \
        --server_port $generator_port \
        --temperature $temperature \
        --top_p $top_p \
        --top_k $top_k \
        --num_samples $num_samples \
        --mem_fraction_static 0.80 \
        --tp_size 4 \
        ${GENERATION_ARGS[@]}

    # Evaluate
    echo "evaluating.."

    python src/evaluation/sglang_eval.py \
    --wandb_name $wandb_name \
    --wandb_id my_eval_run_$RUN_ID \
    --generations_file $generations_file \
    --evaluations_file $evaluations_file \
    --judge_model_path $judge_model_path \
    --server_host $judge_host \
    --server_port $judge_port \
    --context_length $judge_context_length \
    --judge_prompt_file $judge_prompt_file \
    --temperature $temperature \
    --wandb_eval_type $wandb_eval_type \
    --eval_step $step
done