p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=eval_miles_run


cat $0
module load CUDA/12.8
source .venv/bin/activate

# TODO set these
RUN_ID="38875"
CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/${RUN_ID}/checkpoints"
BASE_MODEL="Qwen3-1.7B"
LORA_PATH=""


echo $CKPT_DIR

r
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/13032800/checkpoints/iter_0010000"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_1.7B_ckpt_13032800_generations.jsonl"
evaluations_file="${output_dir}/qwen_1.7B_ckpt_13032800_evaluations.jsonl"

mkdir -p $output_dir

# Evaluation
judge_host="jwb0129i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B_ckpt_13032800"
wandb_tags="lora_run qwen_1.7B"
wandb_eval_type="handcrafted"
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)

        GENERATION_ARGS=(
            --test_cases_file $test_cases_file
            --generations_file $generations_file
            --model_path $HF_DIR/$BASE_MODEL
            --context_length=$context_length
            --server_host $generator_host
            --server_port $generator_port
            --temperature $temperature
            --top_p $top_p
            --top_k $top_k
            --min_p $min_p
            --num_samples $num_samples
            --lora_paths $dir
            --mem_fraction_static 0.80
            --tp_size 4
        )
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)

            --context_length $context_length
            --server_host $generator_host
            --server_port $generator_port
            --temperature $temperature
            --top_p $top_p
            --top_k $top_k
            --min_p $min_p
            --num_samples $num_samples
[content] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)

python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/jobs/mihir/berlin/crowd-pilot/dev_fleet/run_sqwen3-0.6b-sft-torch-crowd-code.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=run_qwen3-1.6b-sft-torch-crowd-code


trap 'echo "Job killed by signal"' USR1

cat $0
module load CUDA/12.8

source .venv/bin/activate

srun sh slurm/jobs/mihir/berlin/crowd-pilot/dev_fleet/qwen_1.7b_no_lora_bs_32.sh