[content] slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run_batch_all.sh (shellscript)

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/handcrafted_${RUN_ID}"

# Local logging configuration (for offline mode)
local_log_dir="data/eval/local_logs/handcrafted"

mkdir -p $output_dir
mkdir -p $local_log_dir

# Evaluation
judge_host="localhost"
judge_port=30000
judge_model_path="/p/scratch/envcomp/mihir/huggingface/Qwen3-30B-A3B"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name=$RUN_NAME
wandb_project="crowd-pilot-miles"
wandb_eval_type="handcrafted-v1"


# Check if this is a lora run or full fine tune
get_latest_iter() {
    ls -1 "$CKPT_DIR" 2>/dev/null | grep -E '^iter_[0-9]+$' | sort -t_ -k2 -n | tail -1
}

latest_iter=$(get_latest_iter)
if [[ -f "$CKPT_DIR/$latest_iter/adapter_model.safetensors" ]]; then
    echo "LoRA adapters found. Evaluating accordingly."
    USE_LORA=true
else
    USE_LORA=false
    echo "No LoRA adapters found. Evaluating as full fine-tune."
fi


# =============================================================================
# PHASE 1: Generate for all checkpoints
# =============================================================================
echo ""
echo "=============================================="
echo "PHASE 1: Generating outputs for all checkpoints"
echo "=============================================="

# Arrays to collect files for batch evaluation
GENERATIONS_FILES=""
EVALUATIONS_FILES=""
EVAL_STEPS=""

# First, generate for base model (step 0)
step=0
generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"

echo "Generating for base model (step 0)..."
python src/evaluation/sglang_generations.py \
    --test_cases_file $test_cases_file \
    --generations_file $generations_file \
    --system_prompt_file "data/prompts/swe_mini_v1.md" \
    --model_path $HF_DIR/$BASE_MODEL \
    --context_length $context_length \
    --server_host $generator_host \
    --server_port $generator_port \
    --temperature $temperature \
    --top_p $top_p \
    --top_k $top_k \
    --min_p $min_p \
    --num_samples $num_samples \
    --max_connections 32 \
    --concurrency 4 \
    --mem_fraction_static 0.85 \
    --tp_size 4

# Add to batch
GENERATIONS_FILES="${generations_file}"
EVALUATIONS_FILES="${evaluations_file}"
EVAL_STEPS="${step}"

# Now generate for each checkpoint
for dir in $(ls -d "${CKPT_DIR}"/iter_*/ | sort -V); do
    step_name=$(basename "$dir")
    step=$((10#${step_name#iter_}))
    generations_file="${output_dir}/${RUN_NAME}_${step}_generations.jsonl"
    evaluations_file="${output_dir}/${RUN_NAME}_${step}_evaluations.jsonl"
    
    if $USE_LORA ; then
        echo "LoRA adapter dir: $dir, Step: $step"
        GENERATION_ARGS=(
            --model_path $HF_DIR/$BASE_MODEL
            --lora_paths $dir
        )
    else
        cp $HF_DIR/$BASE_MODEL/*.json $dir
        MERGED_DIR=${CKPT_DIR}/merged_${step}
        if [ -d $MERGED_DIR ]; then
            echo "Merged checkpoint directory exists. Using for eval: $MERGED_DIR"
        else
            python ../tools/merge_fsdp_checkpoint.py --checkpoint-dir $dir --output-dir $MERGED_DIR --hf-source-dir $HF_DIR/$BASE_MODEL
        fi
        GENERATION_ARGS=(
            --model_path $MERGED_DIR
        )
        echo 'Not using LORA'
    fi

    # Generate
    echo "Generating for step $step..."
    python src/evaluation/sglang_generations.py \
        --test_cases_file $test_cases_file \
        --generations_file $generations_file \
        --system_prompt_file "data/prompts/swe_mini_v1.md" \
        --context_length $context_length \
        --server_host $generator_host \
        --server_port $generator_port \
        --temperature $temperature \
        --top_p $top_p \
        --top_k $top_k \
        --min_p $min_p \
        --num_samples $num_samples \
        --max_connections 32 \
        --concurrency 4 \
        --mem_fraction_static 0.85 \
        --tp_size 4 \
        ${GENERATION_ARGS[@]}

    # Append to batch lists (comma-separated)
    GENERATIONS_FILES="${GENERATIONS_FILES},${generations_file}"
    EVALUATIONS_FILES="${EVALUATIONS_FILES},${evaluations_file}"
    EVAL_STEPS="${EVAL_STEPS},${step}"
done

# =============================================================================
# PHASE 2: Batch evaluation with single judge model load
# =============================================================================
echo ""
echo "=============================================="
echo "PHASE 2: Batch evaluation (single judge load)"
echo "=============================================="

echo "Running batch evaluation with:"
echo "  Generations files: $GENERATIONS_FILES"
echo "  Evaluations files: $EVALUATIONS_FILES"
echo "  Eval steps: $EVAL_STEPS"

python src/evaluation/sglang_eval.py \
    --wandb_name $wandb_name \
    --wandb_id eval_handcrafted_$RUN_ID \
    --generations_files "$GENERATIONS_FILES" \
    --evaluations_files "$EVALUATIONS_FILES" \
    --eval_steps "$EVAL_STEPS" \
    --judge_model_path $judge_model_path \
    --server_host $judge_host \
    --server_port $judge_port \
    --context_length $judge_context_length \
    --judge_prompt_file $judge_prompt_file \
    --temperature $temperature \
    --wandb_eval_type $wandb_eval_type \
    --tp_size 4 \
    --max_connections 32 \
    --concurrency 4 \
    --mem_fraction_static 0.85 \
    --use_local_logger \
    --local_log_dir $local_log_dir

echo ""
echo "=============================================="
echo "Evaluation complete. Local logs saved to:"
echo "  $local_log_dir/eval_$RUN_ID/"
echo ""
echo "To sync to wandb from the login node, run:"
echo "  python src/evaluation/sync_local_logs_to_wandb.py --log_dir $local_log_dir/eval_$RUN_ID"
echo "=============================================="